From 6019d9268ff3b4c8897c077360f016e236bfaade Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Fri, 21 Sep 2018 06:02:58 -0500 Subject: [PATCH] 1. Embed jieba.NET source code. 2. Remove publish profile. 3. Fix docker build --- BotSharp.Algorithm/BotSharp.Algorithm.csproj | 5 +- BotSharp.Core/BotSharp.Core.csproj | 38 +- BotSharp.Core/BotSharp.Core.csproj.user | 6 + .../Engines/Jieba.NET/Common/Counter.cs | 178 +++++++ .../Engines/Jieba.NET/Common/Extensions.cs | 130 +++++ .../Engines/Jieba.NET/Common/FileExtension.cs | 44 ++ .../Engines/Jieba.NET/Common/Trie.cs | 151 ++++++ .../Engines/Jieba.NET/ConfigManager.cs | 63 +++ BotSharp.Core/Engines/Jieba.NET/Constants.cs | 15 + .../Engines/Jieba.NET/DefaultDictionary.cs | 24 + .../Engines/Jieba.NET/FinalSeg/IFinalSeg.cs | 13 + .../Engines/Jieba.NET/FinalSeg/Viterbi.cs | 168 ++++++ .../Engines/Jieba.NET/JiebaSegmenter.cs | 489 ++++++++++++++++++ .../Engines/Jieba.NET/JiebaTagger.cs | 4 + .../Engines/Jieba.NET/JiebaTokenizer.cs | 7 +- BotSharp.Core/Engines/Jieba.NET/Node.cs | 14 + BotSharp.Core/Engines/Jieba.NET/Pair.cs | 19 + .../Engines/Jieba.NET/PosSeg/Pair.cs | 18 + .../Engines/Jieba.NET/PosSeg/PosSegmenter.cs | 301 +++++++++++ .../Engines/Jieba.NET/PosSeg/Viterbi.cs | 176 +++++++ .../Jieba.NET/Spelling/SpellChecker.cs | 158 ++++++ BotSharp.Core/Engines/Jieba.NET/Token.cs | 21 + .../Engines/Jieba.NET/WordDictionary.cs | 136 +++++ BotSharp.NLP/BotSharp.NLP.csproj | 14 +- BotSharp.RestApi/BotSharp.RestApi.csproj | 1 + BotSharp.Vision/BotSharp.Vision.csproj | 5 + BotSharp.Voice/BotSharp.Voice.csproj | 5 + BotSharp.WebHost/BotSharp.WebHost.csproj | 3 +- .../Properties/PublishProfiles/RASA.pubxml | 22 - .../PublishProfiles/RASA.pubxml.user | 11 - BotSharp.sln | 52 +- Dockerfile | 8 +- README.rst | 8 +- 33 files changed, 2176 insertions(+), 131 deletions(-) create mode 100644 BotSharp.Core/BotSharp.Core.csproj.user create mode 100644 BotSharp.Core/Engines/Jieba.NET/Common/Counter.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Common/Extensions.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Common/FileExtension.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Common/Trie.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/ConfigManager.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Constants.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/DefaultDictionary.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/FinalSeg/IFinalSeg.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/FinalSeg/Viterbi.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/JiebaSegmenter.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Node.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Pair.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/PosSeg/Pair.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/PosSeg/PosSegmenter.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/PosSeg/Viterbi.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Spelling/SpellChecker.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/Token.cs create mode 100644 BotSharp.Core/Engines/Jieba.NET/WordDictionary.cs delete mode 100644 BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml delete mode 100644 BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml.user diff --git a/BotSharp.Algorithm/BotSharp.Algorithm.csproj b/BotSharp.Algorithm/BotSharp.Algorithm.csproj index 246b0a3f..34524381 100644 --- a/BotSharp.Algorithm/BotSharp.Algorithm.csproj +++ b/BotSharp.Algorithm/BotSharp.Algorithm.csproj @@ -1,8 +1,8 @@ - + netstandard2.0 - Debug;Release;RASA NLU;DIALOGFLOW;RASA + Debug;Release;RASA;DIALOGFLOW AnyCPU;x64 true https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png @@ -16,6 +16,7 @@ bin\RASA + TRACE;DEBUG;RASA diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index e3e5092c..5440713b 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -6,7 +6,7 @@ SAK SAK AnyCPU;x64 - Debug;Release;RASA NLU;DIALOGFLOW;RASA + Debug;Release;DIALOGFLOW;RASA @@ -37,48 +37,18 @@ If you feel that this project is helpful to you, please Star on the project, we - TRACE;DEBUG - - - - TRACE;DEBUG - - - - TRACE;DEBUG - - - - TRACE;DEBUG - - - - TRACE;DEBUG + TRACE;DEBUG;RASA + bin\RASA - TRACE;MODEL_PER_CONTEXTS - - - - TRACE;MODEL_PER_CONTEXTS + TRACE; - - - - - - - - - - - diff --git a/BotSharp.Core/BotSharp.Core.csproj.user b/BotSharp.Core/BotSharp.Core.csproj.user new file mode 100644 index 00000000..baf24173 --- /dev/null +++ b/BotSharp.Core/BotSharp.Core.csproj.user @@ -0,0 +1,6 @@ + + + + false + + \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/Common/Counter.cs b/BotSharp.Core/Engines/Jieba.NET/Common/Counter.cs new file mode 100644 index 00000000..692613b0 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Common/Counter.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace JiebaNet.Segmenter.Common +{ + public interface ICounter + { + int Count { get; } + int Total { get; } + int this[T key] { get; set; } + IEnumerable> Elements { get; } + + /// + /// Lists the n most common elements from the most common to the least. + /// + /// Number of elements, list all elements if n is less than 0. + /// + IEnumerable> MostCommon(int n = -1); + + /// + /// Subtracts items from a counter. + /// + /// + void Subtract(IEnumerable items); + + /// + /// Subtracts counts from another counter. + /// + /// + void Subtract(ICounter other); + + /// + /// Adds items to a counter. + /// + /// + void Add(IEnumerable items); + + /// + /// Adds another counter. + /// + /// + void Add(ICounter other); + + /// + /// Union is the maximum of value in either of the input . + /// + /// The other counter. + ICounter Union(ICounter other); + + void Remove(T key); + void Clear(); + bool Contains(T key); + } + + public class Counter: ICounter + { + private Dictionary data = new Dictionary(); + + public Counter() {} + + public Counter(IEnumerable items) + { + CountItems(items); + } + + public int Count => data.Count; + public int Total => data.Values.Sum(); + public IEnumerable> Elements => data; + + public int this[T key] + { + get => data.ContainsKey(key) ? data[key] : 0; + set => data[key] = value; + } + + public IEnumerable> MostCommon(int n = -1) + { + var pairs = data.Where(pair => pair.Value > 0).OrderByDescending(pair => pair.Value); + return n < 0 ? pairs : pairs.Take(n); + } + + public void Subtract(IEnumerable items) + { + SubtractItems(items); + } + + public void Subtract(ICounter other) + { + SubtractPairs(other.Elements); + } + + public void Add(IEnumerable items) + { + CountItems(items); + } + + public void Add(ICounter other) + { + CountPairs(other.Elements); + } + + public ICounter Union(ICounter other) + { + var result = new Counter(); + foreach (var pair in data) + { + var count = pair.Value; + var otherCount = other[pair.Key]; + var newCount = count < otherCount ? otherCount : count; + result[pair.Key] = newCount; + } + + foreach (var pair in other.Elements) + { + if (!Contains(pair.Key)) + { + result[pair.Key] = pair.Value; + } + } + return result; + } + + public void Remove(T key) + { + if (data.ContainsKey(key)) + { + data.Remove(key); + } + } + + public void Clear() + { + data.Clear(); + } + + public bool Contains(T key) + { + return data.ContainsKey(key); + } + + #region Private Methods + + private void CountItems(IEnumerable items) + { + foreach (var item in items) + { + data[item] = data.GetDefault(item, 0) + 1; + } + } + + private void CountPairs(IEnumerable> pairs) + { + foreach (var pair in pairs) + { + this[pair.Key] += pair.Value; + } + } + + private void SubtractItems(IEnumerable items) + { + foreach (var item in items) + { + data[item] = data.GetDefault(item, 0) - 1; + } + } + + private void SubtractPairs(IEnumerable> pairs) + { + foreach (var pair in pairs) + { + this[pair.Key] -= pair.Value; + } + } + + #endregion + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/Common/Extensions.cs b/BotSharp.Core/Engines/Jieba.NET/Common/Extensions.cs new file mode 100644 index 00000000..5da7888e --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Common/Extensions.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace JiebaNet.Segmenter.Common +{ + public static class Extensions + { + private static readonly Regex RegexDigits = new Regex(@"\d+", RegexOptions.Compiled); + private static readonly Regex RegexNewline = new Regex("(\r\n|\n|\r)", RegexOptions.Compiled); + + #region Objects + + public static bool IsNull(this object obj) + { + return obj == null; + } + + public static bool IsNotNull(this object obj) + { + return obj != null; + } + + #endregion + + + #region Enumerable + + public static bool IsEmpty(this IEnumerable enumerable) + { + return (enumerable == null) || !enumerable.Any(); + } + + public static bool IsNotEmpty(this IEnumerable enumerable) + { + return (enumerable != null) && enumerable.Any(); + } + + public static TValue GetValueOrDefault(this IDictionary d, TKey key) + { + return d.ContainsKey(key) ? d[key] : default(TValue); + } + + public static TValue GetDefault(this IDictionary dict, TKey key, TValue defaultValue) + { + if (dict.ContainsKey(key)) + { + return dict[key]; + } + return defaultValue; + } + + public static void Update(this IDictionary dict, IDictionary other) + { + foreach (var key in other.Keys) + { + dict[key] = other[key]; + } + } + + #endregion + + #region String & Text + + public static string Left(this string s, int endIndex) + { + if (string.IsNullOrEmpty(s)) + { + return s; + } + + return s.Substring(0, endIndex); + } + + public static string Right(this string s, int startIndex) + { + if (string.IsNullOrEmpty(s)) + { + return s; + } + + + return s.Substring(startIndex); + } + + public static string Sub(this string s, int startIndex, int endIndex) + { + return s.Substring(startIndex, endIndex - startIndex); + } + + public static bool IsInt32(this string s) + { + return RegexDigits.IsMatch(s); + } + + public static string[] SplitLines(this string s) + { + return RegexNewline.Split(s); + } + + public static string Join(this IEnumerable inputs, string separator = ", ") + { + return string.Join(separator, inputs); + } + + public static IEnumerable SubGroupValues(this GroupCollection groups) + { + var result = from Group g in groups + select g.Value; + return result.Skip(1); + } + + #endregion + + #region Conversion + + public static int ToInt32(this char ch) + { + return ch; + } + + public static char ToChar(this int i) + { + return (char)i; + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/Common/FileExtension.cs b/BotSharp.Core/Engines/Jieba.NET/Common/FileExtension.cs new file mode 100644 index 00000000..5983e532 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Common/FileExtension.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.FileProviders; +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Text; + +namespace JiebaNet.Segmenter.Common +{ + public static class FileExtension + { + public static string ReadEmbeddedAllLine(string path) + { + return ReadEmbeddedAllLine(path, Encoding.UTF8); + } + + public static string ReadEmbeddedAllLine(string path,Encoding encoding) + { + using (var sr = new StreamReader(path)) + { + return sr.ReadToEnd(); + } + } + + public static List ReadEmbeddedAllLines(string path, Encoding encoding) + { + List list = new List(); + using (var sr = new StreamReader(path)) + { + string item; + while ((item = sr.ReadLine()) != null) + { + list.Add(item); + } + } + return list; + } + + public static List ReadEmbeddedAllLines(string path) + { + return ReadEmbeddedAllLines(path, Encoding.UTF8); + } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/Common/Trie.cs b/BotSharp.Core/Engines/Jieba.NET/Common/Trie.cs new file mode 100644 index 00000000..3e8c2a5a --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Common/Trie.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace JiebaNet.Segmenter.Common +{ + // Refer to: https://github.com/brianfromoregon/trie + public class TrieNode + { + public char Char { get; set; } + public int Frequency { get; set; } + public Dictionary Children { get; set; } + + public TrieNode(char ch) + { + Char = ch; + Frequency = 0; + + // TODO: or an empty dict? + //Children = null; + } + + public int Insert(string s, int pos, int freq = 1) + { + if (string.IsNullOrEmpty(s) || pos >= s.Length) + { + return 0; + } + + if (Children == null) + { + Children = new Dictionary(); + } + + var c = s[pos]; + if (!Children.ContainsKey(c)) + { + Children[c] = new TrieNode(c); + } + + var curNode = Children[c]; + if (pos == s.Length - 1) + { + curNode.Frequency += freq; + return curNode.Frequency; + } + + return curNode.Insert(s, pos + 1, freq); + } + + public TrieNode Search(string s, int pos) + { + if (string.IsNullOrEmpty(s)) + { + return null; + } + + // if out of range or without any child nodes + if (pos >= s.Length || Children == null) + { + return null; + } + // if reaches the last char of s, it's time to make the decision. + if (pos == s.Length - 1) + { + return Children.ContainsKey(s[pos]) ? Children[s[pos]] : null; + } + // continue if necessary. + return Children.ContainsKey(s[pos]) ? Children[s[pos]].Search(s, pos + 1) : null; + } + } + + public interface ITrie + { + //string BestMatch(string word, long maxTime); + bool Contains(string word); + int Frequency(string word); + int Insert(string word, int freq = 1); + //bool Remove(string word); + int Count { get; } + int TotalFrequency { get; } + } + + public class Trie : ITrie + { + private static readonly char RootChar = '\0'; + + internal TrieNode Root; + + public int Count { get; private set; } + public int TotalFrequency { get; private set; } + + public Trie() + { + Root = new TrieNode(RootChar); + Count = 0; + } + + public bool Contains(string word) + { + CheckWord(word); + + var node = Root.Search(word.Trim(), 0); + return node.IsNotNull() && node.Frequency > 0; + } + + public bool ContainsPrefix(string word) + { + CheckWord(word); + + var node = Root.Search(word.Trim(), 0); + return node.IsNotNull(); + } + + public int Frequency(string word) + { + CheckWord(word); + + var node = Root.Search(word.Trim(), 0); + return node.IsNull() ? 0 : node.Frequency; + } + + public int Insert(string word, int freq = 1) + { + CheckWord(word); + + var i = Root.Insert(word.Trim(), 0, freq); + if (i > 0) + { + TotalFrequency += freq; + Count++; + } + + return i; + } + + public IEnumerable ChildChars(string prefix) + { + var node = Root.Search(prefix.Trim(), 0); + return node.IsNull() || node.Children.IsNull() ? null : node.Children.Select(p => p.Key); + } + + private void CheckWord(string word) + { + if (string.IsNullOrWhiteSpace(word)) + { + throw new ArgumentException("word must not be null or whitespace"); + } + } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/ConfigManager.cs b/BotSharp.Core/Engines/Jieba.NET/ConfigManager.cs new file mode 100644 index 00000000..04583300 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/ConfigManager.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; + +namespace JiebaNet.Segmenter +{ + public class ConfigManager + { + public static string ConfigFileBaseDir + { + get + { + string path = String.Empty; + + var dir = AppDomain.CurrentDomain.GetData("JiebaConfigFileDir"); + if (dir == null) + { + path = "Resources"; + } + else + { + path = Path.Combine(dir.ToString(), "Resources"); + } + + return path; + } + } + + public static string MainDictFile + { + get { return Path.Combine(ConfigFileBaseDir, "dict.txt"); } + } + + public static string ProbTransFile + { + get { return Path.Combine(ConfigFileBaseDir, "prob_trans.json"); } + } + + public static string ProbEmitFile + { + get { return Path.Combine(ConfigFileBaseDir, "prob_emit.json"); } + } + + public static string PosProbStartFile + { + get { return Path.Combine(ConfigFileBaseDir, "pos_prob_start.json"); } + } + + public static string PosProbTransFile + { + get { return Path.Combine(ConfigFileBaseDir, "pos_prob_trans.json"); } + } + + public static string PosProbEmitFile + { + get { return Path.Combine(ConfigFileBaseDir, "pos_prob_emit.json"); } + } + + public static string CharStateTabFile + { + get { return Path.Combine(ConfigFileBaseDir, "char_state_tab.json"); } + } + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/Constants.cs b/BotSharp.Core/Engines/Jieba.NET/Constants.cs new file mode 100644 index 00000000..43151618 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Constants.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Linq; + +namespace JiebaNet.Segmenter +{ + public class Constants + { + public static readonly double MinProb = -3.14e100; + + public static readonly List NounPos = new List() { "n", "ng", "nr", "nrfg", "nrt", "ns", "nt", "nz" }; + public static readonly List VerbPos = new List() { "v", "vd", "vg", "vi", "vn", "vq" }; + public static readonly List NounAndVerbPos = NounPos.Union(VerbPos).ToList(); + public static readonly List IdiomPos = new List() { "i" }; + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/DefaultDictionary.cs b/BotSharp.Core/Engines/Jieba.NET/DefaultDictionary.cs new file mode 100644 index 00000000..45bac668 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/DefaultDictionary.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace JiebaNet.Segmenter +{ + public class DefaultDictionary : Dictionary + { + public new TValue this[TKey key] + { + get + { + if (!ContainsKey(key)) + { + Add(key, default(TValue)); + } + return base[key]; + } + set { base[key] = value; } + } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/FinalSeg/IFinalSeg.cs b/BotSharp.Core/Engines/Jieba.NET/FinalSeg/IFinalSeg.cs new file mode 100644 index 00000000..ed88037f --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/FinalSeg/IFinalSeg.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; + +namespace JiebaNet.Segmenter.FinalSeg +{ + /// + /// ÔڴʵäÇзÖÖ®ºó£¬Ê¹Óô˽ӿڽøÐÐÇз֣¬Ä¬ÈÏʵÏÖΪHMM·½·¨¡£ + /// + public interface IFinalSeg + { + IEnumerable Cut(string sentence); + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/FinalSeg/Viterbi.cs b/BotSharp.Core/Engines/Jieba.NET/FinalSeg/Viterbi.cs new file mode 100644 index 00000000..ca78aa66 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/FinalSeg/Viterbi.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using JiebaNet.Segmenter.Common; +using Newtonsoft.Json; + +namespace JiebaNet.Segmenter.FinalSeg +{ + public class Viterbi : IFinalSeg + { + private static readonly Lazy Lazy = new Lazy(() => new Viterbi()); + private static readonly char[] States = { 'B', 'M', 'E', 'S' }; + + private static readonly Regex RegexChinese = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled); + private static readonly Regex RegexSkip = new Regex(@"([a-zA-Z0-9]+(?:\.\d+)?%?)", RegexOptions.Compiled); + + private static IDictionary> _emitProbs; + private static IDictionary _startProbs; + private static IDictionary> _transProbs; + private static IDictionary _prevStatus; + + private Viterbi() + { + LoadModel(); + } + + // TODO: synchronized + public static Viterbi Instance + { + get { return Lazy.Value; } + } + + public IEnumerable Cut(string sentence) + { + var tokens = new List(); + foreach (var blk in RegexChinese.Split(sentence)) + { + if (RegexChinese.IsMatch(blk)) + { + tokens.AddRange(ViterbiCut(blk)); + } + else + { + var segments = RegexSkip.Split(blk).Where(seg => !string.IsNullOrEmpty(seg)); + tokens.AddRange(segments); + } + } + return tokens; + } + + #region Private Helpers + + private void LoadModel() + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + _prevStatus = new Dictionary() + { + {'B', new []{'E', 'S'}}, + {'M', new []{'M', 'B'}}, + {'S', new []{'S', 'E'}}, + {'E', new []{'B', 'M'}} + }; + + _startProbs = new Dictionary() + { + {'B', -0.26268660809250016}, + {'E', -3.14e+100}, + {'M', -3.14e+100}, + {'S', -1.4652633398537678} + }; + + var transJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.ProbTransFile); + _transProbs = JsonConvert.DeserializeObject>>(transJson); + + var emitJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.ProbEmitFile); + _emitProbs = JsonConvert.DeserializeObject>>(emitJson); + + stopWatch.Stop(); + Debug.WriteLine("model loading finished, time elapsed {0} ms.", stopWatch.ElapsedMilliseconds); + } + + private IEnumerable ViterbiCut(string sentence) + { + var v = new List>(); + IDictionary path = new Dictionary(); + + // Init weights and paths. + v.Add(new Dictionary()); + foreach (var state in States) + { + var emP = _emitProbs[state].GetDefault(sentence[0], Constants.MinProb); + v[0][state] = _startProbs[state] + emP; + path[state] = new Node(state, null); + } + + // For each remaining char + for (var i = 1; i < sentence.Length; ++i) + { + IDictionary vv = new Dictionary(); + v.Add(vv); + IDictionary newPath = new Dictionary(); + foreach (var y in States) + { + var emp = _emitProbs[y].GetDefault(sentence[i], Constants.MinProb); + + Pair candidate = new Pair('\0', double.MinValue); + foreach (var y0 in _prevStatus[y]) + { + var tranp = _transProbs[y0].GetDefault(y, Constants.MinProb); + tranp = v[i - 1][y0] + tranp + emp; + if (candidate.Freq <= tranp) + { + candidate.Freq = tranp; + candidate.Key = y0; + } + } + vv[y] = candidate.Freq; + newPath[y] = new Node(y, path[candidate.Key]); + } + path = newPath; + } + + var probE = v[sentence.Length - 1]['E']; + var probS = v[sentence.Length - 1]['S']; + var finalPath = probE < probS ? path['S'] : path['E']; + + var posList = new List(sentence.Length); + while (finalPath != null) + { + posList.Add(finalPath.Value); + finalPath = finalPath.Parent; + } + posList.Reverse(); + + var tokens = new List(); + int begin = 0, next = 0; + for (var i = 0; i < sentence.Length; i++) + { + var pos = posList[i]; + if (pos == 'B') + begin = i; + else if (pos == 'E') + { + tokens.Add(sentence.Sub(begin, i + 1)); + next = i + 1; + } + else if (pos == 'S') + { + tokens.Add(sentence.Sub(i, i + 1)); + next = i + 1; + } + } + if (next < sentence.Length) + { + tokens.Add(sentence.Substring(next)); + } + + return tokens; + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/JiebaSegmenter.cs b/BotSharp.Core/Engines/Jieba.NET/JiebaSegmenter.cs new file mode 100644 index 00000000..98bd7b39 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/JiebaSegmenter.cs @@ -0,0 +1,489 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using JiebaNet.Segmenter.Common; +using JiebaNet.Segmenter.FinalSeg; + +namespace JiebaNet.Segmenter +{ + public class JiebaSegmenter + { + private static readonly WordDictionary WordDict = WordDictionary.Instance; + private static readonly IFinalSeg FinalSeg = Viterbi.Instance; + private static readonly ISet LoadedPath = new HashSet(); + + private static readonly object locker = new object(); + + internal IDictionary UserWordTagTab { get; set; } + + #region Regular Expressions + + internal static readonly Regex RegexChineseDefault = new Regex(@"([\u4E00-\u9FD5a-zA-Z0-9+#&\._%]+)", RegexOptions.Compiled); + + internal static readonly Regex RegexSkipDefault = new Regex(@"(\r\n|\s)", RegexOptions.Compiled); + + internal static readonly Regex RegexChineseCutAll = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled); + internal static readonly Regex RegexSkipCutAll = new Regex(@"[^a-zA-Z0-9+#\n]", RegexOptions.Compiled); + + internal static readonly Regex RegexEnglishChars = new Regex(@"[a-zA-Z0-9]", RegexOptions.Compiled); + + internal static readonly Regex RegexUserDict = new Regex("^(?.+?)(? [0-9]+)?(? [a-z]+)?$", RegexOptions.Compiled); + + #endregion + + public JiebaSegmenter() + { + UserWordTagTab = new Dictionary(); + } + + /// + /// The main function that segments an entire sentence that contains + /// Chinese characters into seperated words. + /// + /// The string to be segmented. + /// Specify segmentation pattern. True for full pattern, False for accurate pattern. + /// Whether to use the Hidden Markov Model. + /// + public IEnumerable Cut(string text, bool cutAll = false, bool hmm = true) + { + var reHan = RegexChineseDefault; + var reSkip = RegexSkipDefault; + Func> cutMethod = null; + + if (cutAll) + { + reHan = RegexChineseCutAll; + reSkip = RegexSkipCutAll; + } + + if (cutAll) + { + cutMethod = CutAll; + } + else if (hmm) + { + cutMethod = CutDag; + } + else + { + cutMethod = CutDagWithoutHmm; + } + + return CutIt(text, cutMethod, reHan, reSkip, cutAll); + } + + public IEnumerable CutForSearch(string text, bool hmm = true) + { + var result = new List(); + + var words = Cut(text, hmm: hmm); + foreach (var w in words) + { + if (w.Length > 2) + { + foreach (var i in Enumerable.Range(0, w.Length - 1)) + { + var gram2 = w.Substring(i, 2); + if (WordDict.ContainsWord(gram2)) + { + result.Add(gram2); + } + } + } + + if (w.Length > 3) + { + foreach (var i in Enumerable.Range(0, w.Length - 2)) + { + var gram3 = w.Substring(i, 3); + if (WordDict.ContainsWord(gram3)) + { + result.Add(gram3); + } + } + } + + result.Add(w); + } + + return result; + } + + public IEnumerable Tokenize(string text, TokenizerMode mode = TokenizerMode.Default, bool hmm = true) + { + var result = new List(); + + var start = 0; + if (mode == TokenizerMode.Default) + { + foreach (var w in Cut(text, hmm: hmm)) + { + var width = w.Length; + result.Add(new Token(w, start, start + width)); + start += width; + } + } + else + { + foreach (var w in Cut(text, hmm: hmm)) + { + var width = w.Length; + if (width > 2) + { + for (var i = 0; i < width - 1; i++) + { + var gram2 = w.Substring(i, 2); + if (WordDict.ContainsWord(gram2)) + { + result.Add(new Token(gram2, start + i, start + i + 2)); + } + } + } + if (width > 3) + { + for (var i = 0; i < width - 2; i++) + { + var gram3 = w.Substring(i, 3); + if (WordDict.ContainsWord(gram3)) + { + result.Add(new Token(gram3, start + i, start + i + 3)); + } + } + } + + result.Add(new Token(w, start, start + width)); + start += width; + } + } + + return result; + } + + #region Internal Cut Methods + + internal IDictionary> GetDag(string sentence) + { + var dag = new Dictionary>(); + var trie = WordDict.Trie; + + var N = sentence.Length; + for (var k = 0; k < sentence.Length; k++) + { + var templist = new List(); + var i = k; + var frag = sentence.Substring(k, 1); + while (i < N && trie.ContainsKey(frag)) + { + if (trie[frag] > 0) + { + templist.Add(i); + } + + i++; + // TODO: + if (i < N) + { + frag = sentence.Sub(k, i + 1); + } + } + if (templist.Count == 0) + { + templist.Add(k); + } + dag[k] = templist; + } + + return dag; + } + + internal IDictionary> Calc(string sentence, IDictionary> dag) + { + var n = sentence.Length; + var route = new Dictionary>(); + route[n] = new Pair(0, 0.0); + + var logtotal = Math.Log(WordDict.Total); + for (var i = n - 1; i > -1; i--) + { + var candidate = new Pair(-1, double.MinValue); + foreach (int x in dag[i]) + { + var freq = Math.Log(WordDict.GetFreqOrDefault(sentence.Sub(i, x + 1))) - logtotal + route[x + 1].Freq; + if (candidate.Freq < freq) + { + candidate.Freq = freq; + candidate.Key = x; + } + } + route[i] = candidate; + } + return route; + } + + internal IEnumerable CutAll(string sentence) + { + var dag = GetDag(sentence); + + var words = new List(); + var lastPos = -1; + + foreach (var pair in dag) + { + var k = pair.Key; + var nexts = pair.Value; + if (nexts.Count == 1 && k > lastPos) + { + words.Add(sentence.Substring(k, nexts[0] + 1 - k)); + lastPos = nexts[0]; + } + else + { + foreach (var j in nexts) + { + if (j > k) + { + words.Add(sentence.Substring(k, j + 1 - k)); + lastPos = j; + } + } + } + } + + return words; + } + + internal IEnumerable CutDag(string sentence) + { + var dag = GetDag(sentence); + var route = Calc(sentence, dag); + + var tokens = new List(); + + var x = 0; + var n = sentence.Length; + var buf = string.Empty; + while (x < n) + { + var y = route[x].Key + 1; + var w = sentence.Substring(x, y - x); + if (y - x == 1) + { + buf += w; + } + else + { + if (buf.Length > 0) + { + AddBufferToWordList(tokens, buf); + buf = string.Empty; + } + tokens.Add(w); + } + x = y; + } + + if (buf.Length > 0) + { + AddBufferToWordList(tokens, buf); + } + + return tokens; + } + + internal IEnumerable CutDagWithoutHmm(string sentence) + { + var dag = GetDag(sentence); + var route = Calc(sentence, dag); + + var words = new List(); + + var x = 0; + string buf = string.Empty; + var N = sentence.Length; + + var y = -1; + while (x < N) + { + y = route[x].Key + 1; + var l_word = sentence.Substring(x, y - x); + if (RegexEnglishChars.IsMatch(l_word) && l_word.Length == 1) + { + buf += l_word; + x = y; + } + else + { + if (buf.Length > 0) + { + words.Add(buf); + buf = string.Empty; + } + words.Add(l_word); + x = y; + } + } + + if (buf.Length > 0) + { + words.Add(buf); + } + + return words; + } + + internal IEnumerable CutIt(string text, Func> cutMethod, + Regex reHan, Regex reSkip, bool cutAll) + { + var result = new List(); + var blocks = reHan.Split(text); + foreach (var blk in blocks) + { + if (string.IsNullOrEmpty(blk)) + { + continue; + } + + if (reHan.IsMatch(blk)) + { + foreach (var word in cutMethod(blk)) + { + result.Add(word); + } + } + else + { + var tmp = reSkip.Split(blk); + foreach (var x in tmp) + { + if (reSkip.IsMatch(x)) + { + result.Add(x); + } + else if (!cutAll) + { + foreach (var ch in x) + { + result.Add(ch.ToString()); + } + } + else + { + result.Add(x); + } + } + } + } + + return result; + } + + #endregion + + #region Extend Main Dict + + /// + /// Loads user dictionaries. + /// + /// + public void LoadUserDict(string userDictFile) + { + var dictFullPath = Path.GetFullPath(userDictFile); + Debug.WriteLine("Initializing user dictionary: " + userDictFile); + + lock (locker) + { + if (LoadedPath.Contains(dictFullPath)) + return; + + try + { + var startTime = DateTime.Now.Millisecond; + + var lines = File.ReadAllLines(dictFullPath, Encoding.UTF8); + foreach (var line in lines) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var tokens = RegexUserDict.Match(line.Trim()).Groups; + var word = tokens["word"].Value.Trim(); + var freq = tokens["freq"].Value.Trim(); + var tag = tokens["tag"].Value.Trim(); + + var actualFreq = freq.Length > 0 ? int.Parse(freq) : 0; + AddWord(word, actualFreq, tag); + } + + Debug.WriteLine("user dict '{0}' load finished, time elapsed {1} ms", + dictFullPath, DateTime.Now.Millisecond - startTime); + } + catch (IOException e) + { + Debug.Fail(string.Format("'{0}' load failure, reason: {1}", dictFullPath, e.Message)); + } + catch (FormatException fe) + { + Debug.Fail(fe.Message); + } + } + } + + public void AddWord(string word, int freq = 0, string tag = null) + { + if (freq <= 0) + { + freq = WordDict.SuggestFreq(word, Cut(word, hmm: false)); + } + WordDict.AddWord(word, freq); + + // Add user word tag of POS + if (!string.IsNullOrEmpty(tag)) + { + UserWordTagTab[word] = tag; + } + } + + public void DeleteWord(string word) + { + WordDict.DeleteWord(word); + } + + #endregion + + #region Private Helpers + + private void AddBufferToWordList(List words, string buf) + { + if (buf.Length == 1) + { + words.Add(buf); + } + else + { + if (!WordDict.ContainsWord(buf)) + { + var tokens = FinalSeg.Cut(buf); + words.AddRange(tokens); + } + else + { + words.AddRange(buf.Select(ch => ch.ToString())); + } + } + } + + #endregion + } + + public enum TokenizerMode + { + Default, + Search + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs b/BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs index def5b376..30827d16 100644 --- a/BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs +++ b/BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs @@ -2,6 +2,7 @@ using BotSharp.Core.Agents; using BotSharp.NLP; using BotSharp.NLP.Tag; +using JiebaNet.Segmenter; using JiebaNet.Segmenter.PosSeg; using Microsoft.Extensions.Configuration; using System; @@ -39,6 +40,9 @@ namespace BotSharp.Core.Engines.Jieba.NET { if (posSeg == null) { + string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); + AppDomain.CurrentDomain.SetData("JiebaConfigFileDir", contentDir); + posSeg = new PosSegmenter(); } } diff --git a/BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs b/BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs index 9282ab31..de797544 100644 --- a/BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs +++ b/BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs @@ -34,10 +34,13 @@ namespace BotSharp.Core.Engines.Jieba.NET private void Init() { - if(segmenter == null) + if (segmenter == null) { + string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); + AppDomain.CurrentDomain.SetData("JiebaConfigFileDir", contentDir); + segmenter = new JiebaSegmenter(); - segmenter.LoadUserDict($"App_Data{Path.DirectorySeparatorChar}userdict.txt"); + segmenter.LoadUserDict(Path.Combine(contentDir, "userdict.txt")); } } } diff --git a/BotSharp.Core/Engines/Jieba.NET/Node.cs b/BotSharp.Core/Engines/Jieba.NET/Node.cs new file mode 100644 index 00000000..7803d33b --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Node.cs @@ -0,0 +1,14 @@ +namespace JiebaNet.Segmenter +{ + public class Node + { + public char Value { get; private set; } + public Node Parent { get; private set; } + + public Node(char value, Node parent) + { + Value = value; + Parent = parent; + } + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/Pair.cs b/BotSharp.Core/Engines/Jieba.NET/Pair.cs new file mode 100644 index 00000000..59d1d940 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Pair.cs @@ -0,0 +1,19 @@ +namespace JiebaNet.Segmenter +{ + public class Pair + { + public TKey Key { get;set; } + public double Freq { get; set; } + + public Pair(TKey key, double freq) + { + Key = key; + Freq = freq; + } + + public override string ToString() + { + return "Candidate [Key=" + Key + ", Freq=" + Freq + "]"; + } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/PosSeg/Pair.cs b/BotSharp.Core/Engines/Jieba.NET/PosSeg/Pair.cs new file mode 100644 index 00000000..6eda19bb --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/PosSeg/Pair.cs @@ -0,0 +1,18 @@ +namespace JiebaNet.Segmenter.PosSeg +{ + public class Pair + { + public string Word { get; set; } + public string Flag { get; set; } + public Pair(string word, string flag) + { + Word = word; + Flag = flag; + } + + public override string ToString() + { + return string.Format("{0}/{1}", Word, Flag); + } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/PosSeg/PosSegmenter.cs b/BotSharp.Core/Engines/Jieba.NET/PosSeg/PosSegmenter.cs new file mode 100644 index 00000000..7afbe207 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/PosSeg/PosSegmenter.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using JiebaNet.Segmenter.Common; + +namespace JiebaNet.Segmenter.PosSeg +{ + public class PosSegmenter + { + private static readonly WordDictionary WordDict = WordDictionary.Instance; + private static readonly Viterbi PosSeg = Viterbi.Instance; + + // TODO: + private static readonly object locker = new object(); + + #region Regular Expressions + + internal static readonly Regex RegexChineseInternal = new Regex(@"([\u4E00-\u9FD5a-zA-Z0-9+#&\._]+)", RegexOptions.Compiled); + internal static readonly Regex RegexSkipInternal = new Regex(@"(\r\n|\s)", RegexOptions.Compiled); + + internal static readonly Regex RegexChineseDetail = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled); + internal static readonly Regex RegexSkipDetail = new Regex(@"([\.0-9]+|[a-zA-Z0-9]+)", RegexOptions.Compiled); + + internal static readonly Regex RegexEnglishWords = new Regex(@"[a-zA-Z0-9]+", RegexOptions.Compiled); + internal static readonly Regex RegexNumbers = new Regex(@"[\.0-9]+", RegexOptions.Compiled); + + internal static readonly Regex RegexEnglishChar = new Regex(@"^[a-zA-Z0-9]$", RegexOptions.Compiled); + + #endregion + + private static IDictionary _wordTagTab; + + static PosSegmenter() + { + LoadWordTagTab(); + } + + private static void LoadWordTagTab() + { + try + { + _wordTagTab = new Dictionary(); + var lines = FileExtension.ReadEmbeddedAllLines(ConfigManager.MainDictFile); + foreach (var line in lines) + { + var tokens = line.Split(' '); + if (tokens.Length < 2) + { + Debug.Fail(string.Format("Invalid line: {0}", line)); + continue; + } + + var word = tokens[0]; + var tag = tokens[2]; + + _wordTagTab[word] = tag; + } + } + catch (System.IO.IOException e) + { + Debug.Fail(string.Format("Word tag table load failure, reason: {0}", e.Message)); + } + catch (FormatException fe) + { + Debug.Fail(fe.Message); + } + } + + private JiebaSegmenter _segmenter; + + public PosSegmenter() + { + _segmenter = new JiebaSegmenter(); + } + + public PosSegmenter(JiebaSegmenter segmenter) + { + _segmenter = segmenter; + } + + private void CheckNewUserWordTags() + { + if (_segmenter.UserWordTagTab.IsNotEmpty()) + { + _wordTagTab.Update(_segmenter.UserWordTagTab); + _segmenter.UserWordTagTab = new Dictionary(); + } + } + + public IEnumerable Cut(string text, bool hmm = true) + { + return CutInternal(text, hmm); + } + + #region Internal Cut Methods + + internal IEnumerable CutInternal(string text, bool hmm = true) + { + CheckNewUserWordTags(); + + var blocks = RegexChineseInternal.Split(text); + Func> cutMethod = null; + if (hmm) + { + cutMethod = CutDag; + } + else + { + cutMethod = CutDagWithoutHmm; + } + + var tokens = new List(); + foreach (var blk in blocks) + { + if (RegexChineseInternal.IsMatch(blk)) + { + tokens.AddRange(cutMethod(blk)); + } + else + { + var tmp = RegexSkipInternal.Split(blk); + foreach (var x in tmp) + { + if (RegexSkipInternal.IsMatch(x)) + { + tokens.Add(new Pair(x, "x")); + } + else + { + foreach (var xx in x) + { + // TODO: each char? + var xxs = xx.ToString(); + if (RegexNumbers.IsMatch(xxs)) + { + tokens.Add(new Pair(xxs, "m")); + } + else if (RegexEnglishWords.IsMatch(x)) + { + tokens.Add(new Pair(xxs, "eng")); + } + else + { + tokens.Add(new Pair(xxs, "x")); + } + } + } + } + } + } + + return tokens; + } + + internal IEnumerable CutDag(string sentence) + { + var dag = _segmenter.GetDag(sentence); + var route = _segmenter.Calc(sentence, dag); + + var tokens = new List(); + + var x = 0; + var n = sentence.Length; + var buf = string.Empty; + while (x < n) + { + var y = route[x].Key + 1; + var w = sentence.Substring(x, y - x); + if (y - x == 1) + { + buf += w; + } + else + { + if (buf.Length > 0) + { + AddBufferToWordList(tokens, buf); + buf = string.Empty; + } + tokens.Add(new Pair(w, _wordTagTab.GetDefault(w, "x"))); + } + x = y; + } + + if (buf.Length > 0) + { + AddBufferToWordList(tokens, buf); + } + + return tokens; + } + + internal IEnumerable CutDagWithoutHmm(string sentence) + { + var dag = _segmenter.GetDag(sentence); + var route = _segmenter.Calc(sentence, dag); + + var tokens = new List(); + + var x = 0; + var buf = string.Empty; + var n = sentence.Length; + + var y = -1; + while (x < n) + { + y = route[x].Key + 1; + var w = sentence.Substring(x, y - x); + // TODO: char or word? + if (RegexEnglishChar.IsMatch(w)) + { + buf += w; + x = y; + } + else + { + if (buf.Length > 0) + { + tokens.Add(new Pair(buf, "eng")); + buf = string.Empty; + } + tokens.Add(new Pair(w, _wordTagTab.GetDefault(w, "x"))); + x = y; + } + } + + if (buf.Length > 0) + { + tokens.Add(new Pair(buf, "eng")); + } + + return tokens; + } + + internal IEnumerable CutDetail(string text) + { + var tokens = new List(); + var blocks = RegexChineseDetail.Split(text); + foreach (var blk in blocks) + { + if (RegexChineseDetail.IsMatch(blk)) + { + tokens.AddRange(PosSeg.Cut(blk)); + } + else + { + var tmp = RegexSkipDetail.Split(blk); + foreach (var x in tmp) + { + if (!string.IsNullOrWhiteSpace(x)) + { + if (RegexNumbers.IsMatch(x)) + { + tokens.Add(new Pair(x, "m")); + } + else if(RegexEnglishWords.IsMatch(x)) + { + tokens.Add(new Pair(x, "eng")); + } + else + { + tokens.Add(new Pair(x, "x")); + } + } + } + } + } + + return tokens; + } + + #endregion + + #region Private Helpers + + private void AddBufferToWordList(List words, string buf) + { + if (buf.Length == 1) + { + words.Add(new Pair(buf, _wordTagTab.GetDefault(buf, "x"))); + } + else + { + if (!WordDict.ContainsWord(buf)) + { + var tokens = CutDetail(buf); + words.AddRange(tokens); + } + else + { + words.AddRange(buf.Select(ch => new Pair(ch.ToString(), "x"))); + } + } + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/PosSeg/Viterbi.cs b/BotSharp.Core/Engines/Jieba.NET/PosSeg/Viterbi.cs new file mode 100644 index 00000000..349815df --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/PosSeg/Viterbi.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using JiebaNet.Segmenter.Common; +using Newtonsoft.Json; + +namespace JiebaNet.Segmenter.PosSeg +{ + public class Viterbi + { + private static readonly Lazy Lazy = new Lazy(() => new Viterbi()); + + private static IDictionary _startProbs; + private static IDictionary> _transProbs; + private static IDictionary> _emitProbs; + private static IDictionary> _stateTab; + + private Viterbi() + { + LoadModel(); + } + + // TODO: synchronized + public static Viterbi Instance + { + get { return Lazy.Value; } + } + + public IEnumerable Cut(string sentence) + { + var probPosList = ViterbiCut(sentence); + var posList = probPosList.Item2; + + var tokens = new List(); + int begin = 0, next = 0; + for (var i = 0; i < sentence.Length; i++) + { + var parts = posList[i].Split('-'); + var charState = parts[0][0]; + var pos = parts[1]; + if (charState == 'B') + begin = i; + else if (charState == 'E') + { + tokens.Add(new Pair(sentence.Sub(begin, i + 1), pos)); + next = i + 1; + } + else if (charState == 'S') + { + tokens.Add(new Pair(sentence.Sub(i, i + 1), pos)); + next = i + 1; + } + } + if (next < sentence.Length) + { + tokens.Add(new Pair(sentence.Substring(next), posList[next].Split('-')[1])); + } + + return tokens; + } + + #region Private Helpers + + private static void LoadModel() + { + var startJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbStartFile); + _startProbs = JsonConvert.DeserializeObject>(startJson); + + var transJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbTransFile); + _transProbs = JsonConvert.DeserializeObject>>(transJson); + + var emitJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbEmitFile); + _emitProbs = JsonConvert.DeserializeObject>>(emitJson); + + var tabJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.CharStateTabFile); + _stateTab = JsonConvert.DeserializeObject>>(tabJson); + } + + // TODO: change sentence to obs? + private Tuple> ViterbiCut(string sentence) + { + var v = new List>(); + var memPath = new List>(); + + var allStates = _transProbs.Keys.ToList(); + + // Init weights and paths. + v.Add(new Dictionary()); + memPath.Add(new Dictionary()); + foreach (var state in _stateTab.GetDefault(sentence[0], allStates)) + { + var emP = _emitProbs[state].GetDefault(sentence[0], Constants.MinProb); + v[0][state] = _startProbs[state] + emP; + memPath[0][state] = string.Empty; + } + + // For each remaining char + for (var i = 1; i < sentence.Length; ++i) + { + v.Add(new Dictionary()); + memPath.Add(new Dictionary()); + + var prevStates = memPath[i - 1].Keys.Where(k => _transProbs[k].Count > 0); + var curPossibleStates = new HashSet(prevStates.SelectMany(s => _transProbs[s].Keys)); + + IEnumerable obsStates = _stateTab.GetDefault(sentence[i], allStates); + obsStates = curPossibleStates.Intersect(obsStates); + + if (!obsStates.Any()) + { + if (curPossibleStates.Count > 0) + { + obsStates = curPossibleStates; + } + else + { + obsStates = allStates; + } + } + + foreach (var y in obsStates) + { + var emp = _emitProbs[y].GetDefault(sentence[i], Constants.MinProb); + + var prob = double.MinValue; + var state = string.Empty; + + foreach (var y0 in prevStates) + { + var tranp = _transProbs[y0].GetDefault(y, double.MinValue); + tranp = v[i - 1][y0] + tranp + emp; + // TODO: compare two very small values; + // TODO: how to deal with negative infinity + if (prob < tranp || + (prob == tranp && string.Compare(state, y0, StringComparison.CurrentCultureIgnoreCase) < 0)) + { + prob = tranp; + state = y0; + } + } + v[i][y] = prob; + memPath[i][y] = state; + } + } + + var vLast = v.Last(); + var last = memPath.Last().Keys.Select(y => new {State = y, Prob = vLast[y]}); + var endProb = double.MinValue; + var endState = string.Empty; + foreach (var endPoint in last) + { + // TODO: compare two very small values; + if (endProb < endPoint.Prob || + (endProb == endPoint.Prob && String.Compare(endState, endPoint.State, StringComparison.CurrentCultureIgnoreCase) < 0)) + { + endProb = endPoint.Prob; + endState = endPoint.State; + } + } + + var route = new string[sentence.Length]; + var n = sentence.Length - 1; + var curState = endState; + while(n >= 0) + { + route[n] = curState; + curState = memPath[n][curState]; + n--; + } + + return new Tuple>(endProb, route.ToList()); + } + + #endregion + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/Spelling/SpellChecker.cs b/BotSharp.Core/Engines/Jieba.NET/Spelling/SpellChecker.cs new file mode 100644 index 00000000..518f3458 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Spelling/SpellChecker.cs @@ -0,0 +1,158 @@ +using System.Collections.Generic; +using System.Linq; +using JiebaNet.Segmenter.Common; + +namespace JiebaNet.Segmenter.Spelling +{ + public interface ISpellChecker + { + IEnumerable Suggests(string word); + } + + public class SpellChecker : ISpellChecker + { + internal static readonly WordDictionary WordDict = WordDictionary.Instance; + + internal readonly Trie WordTrie; + internal readonly Dictionary> FirstChars; + + public SpellChecker() + { + var wordDict = WordDictionary.Instance; + WordTrie = new Trie(); + FirstChars = new Dictionary>(); + + foreach (var wd in wordDict.Trie) + { + if (wd.Value > 0) + { + WordTrie.Insert(wd.Key, wd.Value); + + if (wd.Key.Length >= 2) + { + var second = wd.Key[1]; + var first = wd.Key[0]; + if (!FirstChars.ContainsKey(second)) + { + FirstChars[second] = new HashSet(); + } + FirstChars[second].Add(first); + } + } + } + } + + internal ISet GetEdits1(string word) + { + var splits = new List(); + for (var i = 0; i <= word.Length; i++) + { + splits.Add(new WordSplit() { Left = word.Substring(0, i), Right = word.Substring(i) }); + } + + var deletes = splits + .Where(s => !string.IsNullOrEmpty(s.Right)) + .Select(s => s.Left + s.Right.Substring(1)); + + var transposes = splits + .Where(s => s.Right.Length > 1) + .Select(s => s.Left + s.Right[1] + s.Right[0] + s.Right.Substring(2)); + + var replaces = new HashSet(); + if (word.Length > 1) + { + var firsts = FirstChars[word[1]]; + foreach (var first in firsts) + { + if (first != word[0]) + { + replaces.Add(first + word.Substring(1)); + } + } + + var node = WordTrie.Root.Children[word[0]]; + for (int i = 1; node.IsNotNull() && node.Children.IsNotEmpty() && i < word.Length; i++) + { + foreach (var c in node.Children.Keys) + { + replaces.Add(word.Substring(0, i) + c + word.Substring(i + 1)); + } + node = node.Children.GetValueOrDefault(word[i]); + } + } + + var inserts = new HashSet(); + if (word.Length > 1) + { + if (FirstChars.ContainsKey(word[0])) + { + var firsts = FirstChars[word[0]]; + foreach (var first in firsts) + { + inserts.Add(first + word); + } + } + + var node = WordTrie.Root.Children.GetValueOrDefault(word[0]); + for (int i = 0; node.IsNotNull() && node.Children.IsNotEmpty() && i < word.Length; i++) + { + foreach (var c in node.Children.Keys) + { + inserts.Add(word.Substring(0, i+1) + c + word.Substring(i+1)); + } + + if (i < word.Length - 1) + { + node = node.Children.GetValueOrDefault(word[i + 1]); + } + } + } + + var result = new HashSet(); + result.UnionWith(deletes); + result.UnionWith(transposes); + result.UnionWith(replaces); + result.UnionWith(inserts); + + return result; + } + + internal ISet GetKnownEdits2(string word) + { + var result = new HashSet(); + foreach (var e1 in GetEdits1(word)) + { + result.UnionWith(GetEdits1(e1).Where(e => WordDictionary.Instance.ContainsWord(e))); + } + return result; + } + + internal ISet GetKnownWords(IEnumerable words) + { + return new HashSet(words.Where(w => WordDictionary.Instance.ContainsWord(w))); + } + + public IEnumerable Suggests(string word) + { + if (WordDict.ContainsWord(word)) + { + return new[] {word}; + } + + var candicates = GetKnownWords(GetEdits1(word)); + if (candicates.IsNotEmpty()) + { + return candicates.OrderByDescending(c => WordDict.GetFreqOrDefault(c)); + } + + candicates.UnionWith(GetKnownEdits2(word)); + return candicates.OrderByDescending(c => WordDict.GetFreqOrDefault(c)); + } + } + + internal class WordSplit + { + public string Left { get; set; } + public string Right { get; set; } + } +} diff --git a/BotSharp.Core/Engines/Jieba.NET/Token.cs b/BotSharp.Core/Engines/Jieba.NET/Token.cs new file mode 100644 index 00000000..10830277 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/Token.cs @@ -0,0 +1,21 @@ +namespace JiebaNet.Segmenter +{ + public class Token + { + public string Word { get; set; } + public int StartIndex { get; set; } + public int EndIndex { get; set; } + + public Token(string word, int startIndex, int endIndex) + { + Word = word; + StartIndex = startIndex; + EndIndex = endIndex; + } + + public override string ToString() + { + return string.Format("[{0}, ({1}, {2})]", Word, StartIndex, EndIndex); + } + } +} \ No newline at end of file diff --git a/BotSharp.Core/Engines/Jieba.NET/WordDictionary.cs b/BotSharp.Core/Engines/Jieba.NET/WordDictionary.cs new file mode 100644 index 00000000..c890b683 --- /dev/null +++ b/BotSharp.Core/Engines/Jieba.NET/WordDictionary.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using JiebaNet.Segmenter.Common; +using Microsoft.Extensions.FileProviders; +using System.Reflection; + +namespace JiebaNet.Segmenter +{ + public class WordDictionary + { + private static readonly Lazy lazy = new Lazy(() => new WordDictionary()); + private static readonly string MainDict = ConfigManager.MainDictFile; + + internal IDictionary Trie = new Dictionary(); + + /// + /// total occurrence of all words. + /// + public double Total { get; set; } + + private WordDictionary() + { + LoadDict(); + + Debug.WriteLine("{0} words (and their prefixes)", Trie.Count); + Debug.WriteLine("total freq: {0}", Total); + } + + public static WordDictionary Instance + { + get { return lazy.Value; } + } + + private void LoadDict() + { + try + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var filePath = ConfigManager.MainDictFile; + + using (var sr = new StreamReader(filePath)) + { + string line = null; + while ((line = sr.ReadLine()) != null) + { + var tokens = line.Split(' '); + if (tokens.Length < 2) + { + Debug.Fail(string.Format("Invalid line: {0}", line)); + continue; + } + + var word = tokens[0]; + var freq = int.Parse(tokens[1]); + + Trie[word] = freq; + Total += freq; + + foreach (var ch in Enumerable.Range(0, word.Length)) + { + var wfrag = word.Sub(0, ch + 1); + if (!Trie.ContainsKey(wfrag)) + { + Trie[wfrag] = 0; + } + } + } + } + + stopWatch.Stop(); + Debug.WriteLine("main dict load finished, time elapsed {0} ms", stopWatch.ElapsedMilliseconds); + } + catch (IOException e) + { + Debug.Fail(string.Format("{0} load failure, reason: {1}", MainDict, e.Message)); + } + catch (FormatException fe) + { + Debug.Fail(fe.Message); + } + } + + public bool ContainsWord(string word) + { + return Trie.ContainsKey(word) && Trie[word] > 0; + } + + public int GetFreqOrDefault(string key) + { + if (ContainsWord(key)) + return Trie[key]; + else + return 1; + } + + public void AddWord(string word, int freq, string tag = null) + { + if (ContainsWord(word)) + { + Total -= Trie[word]; + } + + Trie[word] = freq; + Total += freq; + for (var i = 0; i < word.Length; i++) + { + var wfrag = word.Substring(0, i + 1); + if (!Trie.ContainsKey(wfrag)) + { + Trie[wfrag] = 0; + } + } + } + + public void DeleteWord(string word) + { + AddWord(word, 0); + } + + internal int SuggestFreq(string word, IEnumerable segments) + { + double freq = 1; + foreach (var seg in segments) + { + freq *= GetFreqOrDefault(seg) / Total; + } + + return Math.Max((int)(freq * Total) + 1, GetFreqOrDefault(word)); + } + } +} \ No newline at end of file diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index 11c2572b..41185d49 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -31,15 +31,11 @@ Naive Bayes Classifier - DEBUG;TRACE - - - - DEBUG;TRACE + RASA;DEBUG;TRACE + bin\RASA - @@ -47,10 +43,4 @@ Naive Bayes Classifier - - - Code - - - diff --git a/BotSharp.RestApi/BotSharp.RestApi.csproj b/BotSharp.RestApi/BotSharp.RestApi.csproj index 480c11a9..5094ccc5 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.csproj +++ b/BotSharp.RestApi/BotSharp.RestApi.csproj @@ -27,6 +27,7 @@ BotSharp.RestApi.xml TRACE;DEBUG + bin\RASA diff --git a/BotSharp.Vision/BotSharp.Vision.csproj b/BotSharp.Vision/BotSharp.Vision.csproj index 9f5c4f4a..3982f9c9 100644 --- a/BotSharp.Vision/BotSharp.Vision.csproj +++ b/BotSharp.Vision/BotSharp.Vision.csproj @@ -2,6 +2,11 @@ netstandard2.0 + Debug;Release;RASA;DIALOGFLOW + + + + bin\RASA diff --git a/BotSharp.Voice/BotSharp.Voice.csproj b/BotSharp.Voice/BotSharp.Voice.csproj index 9f5c4f4a..3982f9c9 100644 --- a/BotSharp.Voice/BotSharp.Voice.csproj +++ b/BotSharp.Voice/BotSharp.Voice.csproj @@ -2,6 +2,11 @@ netstandard2.0 + Debug;Release;RASA;DIALOGFLOW + + + + bin\RASA diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index a46d91fe..b72f6cc6 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -12,7 +12,8 @@ - TRACE + DEBUG;TRACE;RASA + bin\RASA diff --git a/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml b/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml deleted file mode 100644 index 543b973e..00000000 --- a/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - FileSystem - FileSystem - RASA - Any CPU - - True - False - netcoreapp2.1 - 03dca427-327a-4fc9-9a2f-57d17f16708c - false - <_IsPortable>true - PublishOutput - True - - \ No newline at end of file diff --git a/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml.user b/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml.user deleted file mode 100644 index 86910c09..00000000 --- a/BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml.user +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - <_PublishTargetUrl>D:\Projects\BotSharp\BotSharp.WebHost\PublishOutput - - \ No newline at end of file diff --git a/BotSharp.sln b/BotSharp.sln index b9359cea..ae3fe396 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -8,17 +8,17 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.RestApi", "BotSharp.RestApi\BotSharp.RestApi.csproj", "{80DDAA05-69BC-49DD-96A4-37345E7A2E20}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotSharp.WebHost\BotSharp.WebHost.csproj", "{03DCA427-327A-4FC9-9A2F-57D17F16708C}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.UnitTest", "BotSharp.Core.UnitTest\BotSharp.Core.UnitTest.csproj", "{A31A6853-DFB8-477D-8F09-8E6E3D166102}" + ProjectSection(ProjectDependencies) = postProject + {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF} = {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF} + {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22} = {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22} + {75B02A7C-EDB3-4082-9C1F-471773E760C3} = {75B02A7C-EDB3-4082-9C1F-471773E760C3} + {184F8B93-68C2-4849-9625-4023C4360990} = {184F8B93-68C2-4849-9625-4023C4360990} + EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Algorithm", "BotSharp.Algorithm\BotSharp.Algorithm.csproj", "{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Algorithm.UnitTest", "BotSharp.Algorithm.UnitTest\BotSharp.Algorithm.UnitTest.csproj", "{4C4EE7A8-99CA-41FE-9517-C66799720784}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NLP\BotSharp.NLP.csproj", "{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Voice", "BotSharp.Voice\BotSharp.Voice.csproj", "{184F8B93-68C2-4849-9625-4023C4360990}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Vision", "BotSharp.Vision\BotSharp.Vision.csproj", "{75B02A7C-EDB3-4082-9C1F-471773E760C3}" @@ -55,14 +55,6 @@ Global {03DCA427-327A-4FC9-9A2F-57D17F16708C}.RASA|Any CPU.Build.0 = RASA|Any CPU {03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.ActiveCfg = Release|Any CPU {03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.Build.0 = Release|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.RASA|Any CPU.ActiveCfg = RASA|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.RASA|Any CPU.Build.0 = RASA|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.Build.0 = Release|Any CPU {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Debug|Any CPU.Build.0 = Debug|Any CPU {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU @@ -71,14 +63,6 @@ Global {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.RASA|Any CPU.Build.0 = RASA|Any CPU {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Release|Any CPU.ActiveCfg = Release|Any CPU {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Release|Any CPU.Build.0 = Release|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.RASA|Any CPU.ActiveCfg = RASA|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.RASA|Any CPU.Build.0 = RASA|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4C4EE7A8-99CA-41FE-9517-C66799720784}.Release|Any CPU.Build.0 = Release|Any CPU {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Debug|Any CPU.Build.0 = Debug|Any CPU {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU @@ -87,28 +71,20 @@ Global {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.RASA|Any CPU.Build.0 = RASA|Any CPU {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Release|Any CPU.ActiveCfg = Release|Any CPU {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Release|Any CPU.Build.0 = Release|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.RASA|Any CPU.ActiveCfg = RASA|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.RASA|Any CPU.Build.0 = RASA|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Release|Any CPU.Build.0 = Release|Any CPU {184F8B93-68C2-4849-9625-4023C4360990}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {184F8B93-68C2-4849-9625-4023C4360990}.Debug|Any CPU.Build.0 = Debug|Any CPU - {184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.ActiveCfg = Debug|Any CPU - {184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.Build.0 = Debug|Any CPU - {184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.ActiveCfg = Release|Any CPU - {184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.Build.0 = Release|Any CPU + {184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU + {184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU + {184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.ActiveCfg = RASA|Any CPU + {184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.Build.0 = RASA|Any CPU {184F8B93-68C2-4849-9625-4023C4360990}.Release|Any CPU.ActiveCfg = Release|Any CPU {184F8B93-68C2-4849-9625-4023C4360990}.Release|Any CPU.Build.0 = Release|Any CPU {75B02A7C-EDB3-4082-9C1F-471773E760C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {75B02A7C-EDB3-4082-9C1F-471773E760C3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.ActiveCfg = Debug|Any CPU - {75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.Build.0 = Debug|Any CPU - {75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.ActiveCfg = Release|Any CPU - {75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.Build.0 = Release|Any CPU + {75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU + {75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU + {75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.ActiveCfg = RASA|Any CPU + {75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.Build.0 = RASA|Any CPU {75B02A7C-EDB3-4082-9C1F-471773E760C3}.Release|Any CPU.ActiveCfg = Release|Any CPU {75B02A7C-EDB3-4082-9C1F-471773E760C3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection diff --git a/Dockerfile b/Dockerfile index 8cb96e19..6c205b90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,17 +4,13 @@ WORKDIR /source # copies the rest of your code COPY . . -RUN dotnet build -RUN dotnet publish /p:PublishProfile=RASA /p:Configuration=RASA --output /app +# RUN dotnet build +RUN dotnet publish BotSharp.WebHost/BotSharp.WebHost.csproj --configuration RASA --output /app # copy Settings folder WORKDIR /app COPY Settings Settings RUN mkdir App_Data/Projects -# move data for jieba.NetCore -# RUN mv App_Data/Resources Resources -# RUN mv App_Data/userdict.txt userdict.txt - # stage 2: run ENTRYPOINT [ "dotnet", "BotSharp.WebHost.dll" ] diff --git a/README.rst b/README.rst index d30ef635..1a09600b 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ The Open Source AI Bot Platform Builder ====================================================== - + .. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg :target: `gitter`_ :alt: gitter @@ -18,8 +18,10 @@ The Open Source AI Bot Platform Builder .. image:: https://img.shields.io/nuget/dt/EntityFrameworkCore.BootKit.svg :target: `botsharpnuget`_ :alt: NuGet - - + +This project is for learning purposes only, please do not use it in a production environment. +********************************************************************************************** + *"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our .NET developers using the BotSharp AI BOT Platform Builder to build a CaaP. It opens up as much learning power as possible for your own robots and precisely control every step of the AI processing pipeline."* **BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.