SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

1. Embed jieba.NET source code.

Browse source 
2. Remove publish profile.
3. Fix docker build
This commit is contained in:
Oceania2018 2018-09-21 06:02:58 -05:00
parent 6eeb617dbf
commit 6019d9268f
33 changed files with 2176 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
BotSharp.Algorithm/BotSharp.Algorithm.csproj
View file

@ -1,8 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Configurations>Debug;Release;RASA NLU;DIALOGFLOW;RASA</Configurations>
<Configurations>Debug;Release;RASA;DIALOGFLOW</Configurations>
<Platforms>AnyCPU;x64</Platforms>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<PackageIconUrl>https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png</PackageIconUrl>
@ -16,6 +16,7 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
<OutputPath>bin\RASA</OutputPath>
<DefineConstants>TRACE;DEBUG;RASA</DefineConstants>
</PropertyGroup>
</Project>

38
BotSharp.Core/BotSharp.Core.csproj
View file

@ -6,7 +6,7 @@
<SccAuxPath>SAK</SccAuxPath>
<SccLocalPath>SAK</SccLocalPath>
<Platforms>AnyCPU;x64</Platforms>
<Configurations>Debug;Release;RASA NLU;DIALOGFLOW;RASA</Configurations>
<Configurations>Debug;Release;DIALOGFLOW;RASA</Configurations>
</PropertyGroup>
<PropertyGroup>
@ -37,48 +37,18 @@ If you feel that this project is helpful to you, please Star on the project, we
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
<DefineConstants>TRACE;DEBUG</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA NLU|AnyCPU'">
<DefineConstants>TRACE;DEBUG</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<DefineConstants>TRACE;DEBUG</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|x64'">
<DefineConstants>TRACE;DEBUG</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA NLU|x64'">
<DefineConstants>TRACE;DEBUG</DefineConstants>
<DefineConstants>TRACE;DEBUG;RASA</DefineConstants>
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<DefineConstants>TRACE;MODEL_PER_CONTEXTS</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<DefineConstants>TRACE;MODEL_PER_CONTEXTS</DefineConstants>
<DefineConstants>TRACE;</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Accounts\**" />
<Compile Remove="Engines\CoreNlp\**" />
<EmbeddedResource Remove="Accounts\**" />
<EmbeddedResource Remove="Engines\CoreNlp\**" />
<None Remove="Accounts\**" />
<None Remove="Engines\CoreNlp\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BotSharp.NLP" Version="0.3.0" />
<PackageReference Include="Colorful.Console" Version="1.2.9" />
<PackageReference Include="DotNetToolkit" Version="1.6.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.9.1" />
<PackageReference Include="JiebaNet.Segmenter" Version="1.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="RestSharp" Version="106.3.1" />

6
BotSharp.Core/BotSharp.Core.csproj.user Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ShowAllFiles>false</ShowAllFiles>
</PropertyGroup>
</Project>

178
BotSharp.Core/Engines/Jieba.NET/Common/Counter.cs Normal file
View file

@ -0,0 +1,178 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace JiebaNet.Segmenter.Common
{
public interface ICounter<T>
{
int Count { get; }
int Total { get; }
int this[T key] { get; set; }
IEnumerable<KeyValuePair<T, int>> Elements { get; }
/// <summary>
/// Lists the n most common elements from the most common to the least.
/// </summary>
/// <param name="n">Number of elements, list all elements if n is less than 0.</param>
/// <returns></returns>
IEnumerable<KeyValuePair<T, int>> MostCommon(int n = -1);
/// <summary>
/// Subtracts items from a counter.
/// </summary>
/// <param name="items"></param>
void Subtract(IEnumerable<T> items);
/// <summary>
/// Subtracts counts from another counter.
/// </summary>
/// <param name="other"></param>
void Subtract(ICounter<T> other);
/// <summary>
/// Adds items to a counter.
/// </summary>
/// <param name="items"></param>
void Add(IEnumerable<T> items);
/// <summary>
/// Adds another counter.
/// </summary>
/// <param name="other"></param>
void Add(ICounter<T> other);
/// <summary>
/// Union is the maximum of value in either of the input <see cref="ICounter{T}"/>.
/// </summary>
/// <param name="other">The other counter.</param>
ICounter<T> Union(ICounter<T> other);
void Remove(T key);
void Clear();
bool Contains(T key);
}
public class Counter<T>: ICounter<T>
{
private Dictionary<T, int> data = new Dictionary<T, int>();
public Counter() {}
public Counter(IEnumerable<T> items)
{
CountItems(items);
}
public int Count => data.Count;
public int Total => data.Values.Sum();
public IEnumerable<KeyValuePair<T, int>> Elements => data;
public int this[T key]
{
get => data.ContainsKey(key) ? data[key] : 0;
set => data[key] = value;
}
public IEnumerable<KeyValuePair<T, int>> MostCommon(int n = -1)
{
var pairs = data.Where(pair => pair.Value > 0).OrderByDescending(pair => pair.Value);
return n < 0 ? pairs : pairs.Take(n);
}
public void Subtract(IEnumerable<T> items)
{
SubtractItems(items);
}
public void Subtract(ICounter<T> other)
{
SubtractPairs(other.Elements);
}
public void Add(IEnumerable<T> items)
{
CountItems(items);
}
public void Add(ICounter<T> other)
{
CountPairs(other.Elements);
}
public ICounter<T> Union(ICounter<T> other)
{
var result = new Counter<T>();
foreach (var pair in data)
{
var count = pair.Value;
var otherCount = other[pair.Key];
var newCount = count < otherCount ? otherCount : count;
result[pair.Key] = newCount;
}
foreach (var pair in other.Elements)
{
if (!Contains(pair.Key))
{
result[pair.Key] = pair.Value;
}
}
return result;
}
public void Remove(T key)
{
if (data.ContainsKey(key))
{
data.Remove(key);
}
}
public void Clear()
{
data.Clear();
}
public bool Contains(T key)
{
return data.ContainsKey(key);
}
#region Private Methods
private void CountItems(IEnumerable<T> items)
{
foreach (var item in items)
{
data[item] = data.GetDefault(item, 0) + 1;
}
}
private void CountPairs(IEnumerable<KeyValuePair<T, int>> pairs)
{
foreach (var pair in pairs)
{
this[pair.Key] += pair.Value;
}
}
private void SubtractItems(IEnumerable<T> items)
{
foreach (var item in items)
{
data[item] = data.GetDefault(item, 0) - 1;
}
}
private void SubtractPairs(IEnumerable<KeyValuePair<T, int>> pairs)
{
foreach (var pair in pairs)
{
this[pair.Key] -= pair.Value;
}
}
#endregion
}
}

130
BotSharp.Core/Engines/Jieba.NET/Common/Extensions.cs Normal file
View file

@ -0,0 +1,130 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace JiebaNet.Segmenter.Common
{
public static class Extensions
{
private static readonly Regex RegexDigits = new Regex(@"\d+", RegexOptions.Compiled);
private static readonly Regex RegexNewline = new Regex("(\r\n|\n|\r)", RegexOptions.Compiled);
#region Objects
public static bool IsNull(this object obj)
{
return obj == null;
}
public static bool IsNotNull(this object obj)
{
return obj != null;
}
#endregion
#region Enumerable
public static bool IsEmpty<T>(this IEnumerable<T> enumerable)
{
return (enumerable == null) || !enumerable.Any();
}
public static bool IsNotEmpty<T>(this IEnumerable<T> enumerable)
{
return (enumerable != null) && enumerable.Any();
}
public static TValue GetValueOrDefault<TKey, TValue>(this IDictionary<TKey, TValue> d, TKey key)
{
return d.ContainsKey(key) ? d[key] : default(TValue);
}
public static TValue GetDefault<TKey, TValue>(this IDictionary<TKey, TValue> dict, TKey key, TValue defaultValue)
{
if (dict.ContainsKey(key))
{
return dict[key];
}
return defaultValue;
}
public static void Update<TKey, TValue>(this IDictionary<TKey, TValue> dict, IDictionary<TKey, TValue> other)
{
foreach (var key in other.Keys)
{
dict[key] = other[key];
}
}
#endregion
#region String & Text
public static string Left(this string s, int endIndex)
{
if (string.IsNullOrEmpty(s))
{
return s;
}
return s.Substring(0, endIndex);
}
public static string Right(this string s, int startIndex)
{
if (string.IsNullOrEmpty(s))
{
return s;
}
return s.Substring(startIndex);
}
public static string Sub(this string s, int startIndex, int endIndex)
{
return s.Substring(startIndex, endIndex - startIndex);
}
public static bool IsInt32(this string s)
{
return RegexDigits.IsMatch(s);
}
public static string[] SplitLines(this string s)
{
return RegexNewline.Split(s);
}
public static string Join(this IEnumerable<string> inputs, string separator = ", ")
{
return string.Join(separator, inputs);
}
public static IEnumerable<string> SubGroupValues(this GroupCollection groups)
{
var result = from Group g in groups
select g.Value;
return result.Skip(1);
}
#endregion
#region Conversion
public static int ToInt32(this char ch)
{
return ch;
}
public static char ToChar(this int i)
{
return (char)i;
}
#endregion
}
}

44
BotSharp.Core/Engines/Jieba.NET/Common/FileExtension.cs Normal file
View file

@ -0,0 +1,44 @@
using Microsoft.Extensions.FileProviders;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Text;
namespace JiebaNet.Segmenter.Common
{
public static class FileExtension
{
public static string ReadEmbeddedAllLine(string path)
{
return ReadEmbeddedAllLine(path, Encoding.UTF8);
}
public static string ReadEmbeddedAllLine(string path,Encoding encoding)
{
using (var sr = new StreamReader(path))
{
return sr.ReadToEnd();
}
}
public static List<string> ReadEmbeddedAllLines(string path, Encoding encoding)
{
List<string> list = new List<string>();
using (var sr = new StreamReader(path))
{
string item;
while ((item = sr.ReadLine()) != null)
{
list.Add(item);
}
}
return list;
}
public static List<string> ReadEmbeddedAllLines(string path)
{
return ReadEmbeddedAllLines(path, Encoding.UTF8);
}
}
}

151
BotSharp.Core/Engines/Jieba.NET/Common/Trie.cs Normal file
View file

@ -0,0 +1,151 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace JiebaNet.Segmenter.Common
{
// Refer to: https://github.com/brianfromoregon/trie
public class TrieNode
{
public char Char { get; set; }
public int Frequency { get; set; }
public Dictionary<char, TrieNode> Children { get; set; }
public TrieNode(char ch)
{
Char = ch;
Frequency = 0;
// TODO: or an empty dict?
//Children = null;
}
public int Insert(string s, int pos, int freq = 1)
{
if (string.IsNullOrEmpty(s) || pos >= s.Length)
{
return 0;
}
if (Children == null)
{
Children = new Dictionary<char, TrieNode>();
}
var c = s[pos];
if (!Children.ContainsKey(c))
{
Children[c] = new TrieNode(c);
}
var curNode = Children[c];
if (pos == s.Length - 1)
{
curNode.Frequency += freq;
return curNode.Frequency;
}
return curNode.Insert(s, pos + 1, freq);
}
public TrieNode Search(string s, int pos)
{
if (string.IsNullOrEmpty(s))
{
return null;
}
// if out of range or without any child nodes
if (pos >= s.Length || Children == null)
{
return null;
}
// if reaches the last char of s, it's time to make the decision.
if (pos == s.Length - 1)
{
return Children.ContainsKey(s[pos]) ? Children[s[pos]] : null;
}
// continue if necessary.
return Children.ContainsKey(s[pos]) ? Children[s[pos]].Search(s, pos + 1) : null;
}
}
public interface ITrie
{
//string BestMatch(string word, long maxTime);
bool Contains(string word);
int Frequency(string word);
int Insert(string word, int freq = 1);
//bool Remove(string word);
int Count { get; }
int TotalFrequency { get; }
}
public class Trie : ITrie
{
private static readonly char RootChar = '\0';
internal TrieNode Root;
public int Count { get; private set; }
public int TotalFrequency { get; private set; }
public Trie()
{
Root = new TrieNode(RootChar);
Count = 0;
}
public bool Contains(string word)
{
CheckWord(word);
var node = Root.Search(word.Trim(), 0);
return node.IsNotNull() && node.Frequency > 0;
}
public bool ContainsPrefix(string word)
{
CheckWord(word);
var node = Root.Search(word.Trim(), 0);
return node.IsNotNull();
}
public int Frequency(string word)
{
CheckWord(word);
var node = Root.Search(word.Trim(), 0);
return node.IsNull() ? 0 : node.Frequency;
}
public int Insert(string word, int freq = 1)
{
CheckWord(word);
var i = Root.Insert(word.Trim(), 0, freq);
if (i > 0)
{
TotalFrequency += freq;
Count++;
}
return i;
}
public IEnumerable<char> ChildChars(string prefix)
{
var node = Root.Search(prefix.Trim(), 0);
return node.IsNull() || node.Children.IsNull() ? null : node.Children.Select(p => p.Key);
}
private void CheckWord(string word)
{
if (string.IsNullOrWhiteSpace(word))
{
throw new ArgumentException("word must not be null or whitespace");
}
}
}
}

63
BotSharp.Core/Engines/Jieba.NET/ConfigManager.cs Normal file
View file

@ -0,0 +1,63 @@
using System;
using System.IO;
namespace JiebaNet.Segmenter
{
public class ConfigManager
{
public static string ConfigFileBaseDir
{
get
{
string path = String.Empty;
var dir = AppDomain.CurrentDomain.GetData("JiebaConfigFileDir");
if (dir == null)
{
path = "Resources";
}
else
{
path = Path.Combine(dir.ToString(), "Resources");
}
return path;
}
}
public static string MainDictFile
{
get { return Path.Combine(ConfigFileBaseDir, "dict.txt"); }
}
public static string ProbTransFile
{
get { return Path.Combine(ConfigFileBaseDir, "prob_trans.json"); }
}
public static string ProbEmitFile
{
get { return Path.Combine(ConfigFileBaseDir, "prob_emit.json"); }
}
public static string PosProbStartFile
{
get { return Path.Combine(ConfigFileBaseDir, "pos_prob_start.json"); }
}
public static string PosProbTransFile
{
get { return Path.Combine(ConfigFileBaseDir, "pos_prob_trans.json"); }
}
public static string PosProbEmitFile
{
get { return Path.Combine(ConfigFileBaseDir, "pos_prob_emit.json"); }
}
public static string CharStateTabFile
{
get { return Path.Combine(ConfigFileBaseDir, "char_state_tab.json"); }
}
}
}

15
BotSharp.Core/Engines/Jieba.NET/Constants.cs Normal file
View file

@ -0,0 +1,15 @@
using System.Collections.Generic;
using System.Linq;
namespace JiebaNet.Segmenter
{
public class Constants
{
public static readonly double MinProb = -3.14e100;
public static readonly List<string> NounPos = new List<string>() { "n", "ng", "nr", "nrfg", "nrt", "ns", "nt", "nz" };
public static readonly List<string> VerbPos = new List<string>() { "v", "vd", "vg", "vi", "vn", "vq" };
public static readonly List<string> NounAndVerbPos = NounPos.Union(VerbPos).ToList();
public static readonly List<string> IdiomPos = new List<string>() { "i" };
}
}

24
BotSharp.Core/Engines/Jieba.NET/DefaultDictionary.cs Normal file
View file

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace JiebaNet.Segmenter
{
public class DefaultDictionary<TKey, TValue> : Dictionary<TKey, TValue>
{
public new TValue this[TKey key]
{
get
{
if (!ContainsKey(key))
{
Add(key, default(TValue));
}
return base[key];
}
set { base[key] = value; }
}
}
}

13
BotSharp.Core/Engines/Jieba.NET/FinalSeg/IFinalSeg.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
namespace JiebaNet.Segmenter.FinalSeg
{
/// <summary>
/// 在词典切分之后，使用此接口进行切分，默认实现为HMM方法。
/// </summary>
public interface IFinalSeg
{
IEnumerable<string> Cut(string sentence);
}
}

168
BotSharp.Core/Engines/Jieba.NET/FinalSeg/Viterbi.cs Normal file
View file

@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using JiebaNet.Segmenter.Common;
using Newtonsoft.Json;
namespace JiebaNet.Segmenter.FinalSeg
{
public class Viterbi : IFinalSeg
{
private static readonly Lazy<Viterbi> Lazy = new Lazy<Viterbi>(() => new Viterbi());
private static readonly char[] States = { 'B', 'M', 'E', 'S' };
private static readonly Regex RegexChinese = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled);
private static readonly Regex RegexSkip = new Regex(@"([a-zA-Z0-9]+(?:\.\d+)?%?)", RegexOptions.Compiled);
private static IDictionary<char, IDictionary<char, double>> _emitProbs;
private static IDictionary<char, double> _startProbs;
private static IDictionary<char, IDictionary<char, double>> _transProbs;
private static IDictionary<char, char[]> _prevStatus;
private Viterbi()
{
LoadModel();
}
// TODO: synchronized
public static Viterbi Instance
{
get { return Lazy.Value; }
}
public IEnumerable<string> Cut(string sentence)
{
var tokens = new List<string>();
foreach (var blk in RegexChinese.Split(sentence))
{
if (RegexChinese.IsMatch(blk))
{
tokens.AddRange(ViterbiCut(blk));
}
else
{
var segments = RegexSkip.Split(blk).Where(seg => !string.IsNullOrEmpty(seg));
tokens.AddRange(segments);
}
}
return tokens;
}
#region Private Helpers
private void LoadModel()
{
var stopWatch = new Stopwatch();
stopWatch.Start();
_prevStatus = new Dictionary<char, char[]>()
{
{'B', new []{'E', 'S'}},
{'M', new []{'M', 'B'}},
{'S', new []{'S', 'E'}},
{'E', new []{'B', 'M'}}
};
_startProbs = new Dictionary<char, double>()
{
{'B', -0.26268660809250016},
{'E', -3.14e+100},
{'M', -3.14e+100},
{'S', -1.4652633398537678}
};
var transJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.ProbTransFile);
_transProbs = JsonConvert.DeserializeObject<IDictionary<char, IDictionary<char, double>>>(transJson);
var emitJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.ProbEmitFile);
_emitProbs = JsonConvert.DeserializeObject<IDictionary<char, IDictionary<char, double>>>(emitJson);
stopWatch.Stop();
Debug.WriteLine("model loading finished, time elapsed {0} ms.", stopWatch.ElapsedMilliseconds);
}
private IEnumerable<string> ViterbiCut(string sentence)
{
var v = new List<IDictionary<char, double>>();
IDictionary<char, Node> path = new Dictionary<char, Node>();
// Init weights and paths.
v.Add(new Dictionary<char, double>());
foreach (var state in States)
{
var emP = _emitProbs[state].GetDefault(sentence[0], Constants.MinProb);
v[0][state] = _startProbs[state] + emP;
path[state] = new Node(state, null);
}
// For each remaining char
for (var i = 1; i < sentence.Length; ++i)
{
IDictionary<char, double> vv = new Dictionary<char, double>();
v.Add(vv);
IDictionary<char, Node> newPath = new Dictionary<char, Node>();
foreach (var y in States)
{
var emp = _emitProbs[y].GetDefault(sentence[i], Constants.MinProb);
Pair<char> candidate = new Pair<char>('\0', double.MinValue);
foreach (var y0 in _prevStatus[y])
{
var tranp = _transProbs[y0].GetDefault(y, Constants.MinProb);
tranp = v[i - 1][y0] + tranp + emp;
if (candidate.Freq <= tranp)
{
candidate.Freq = tranp;
candidate.Key = y0;
}
}
vv[y] = candidate.Freq;
newPath[y] = new Node(y, path[candidate.Key]);
}
path = newPath;
}
var probE = v[sentence.Length - 1]['E'];
var probS = v[sentence.Length - 1]['S'];
var finalPath = probE < probS ? path['S'] : path['E'];
var posList = new List<char>(sentence.Length);
while (finalPath != null)
{
posList.Add(finalPath.Value);
finalPath = finalPath.Parent;
}
posList.Reverse();
var tokens = new List<string>();
int begin = 0, next = 0;
for (var i = 0; i < sentence.Length; i++)
{
var pos = posList[i];
if (pos == 'B')
begin = i;
else if (pos == 'E')
{
tokens.Add(sentence.Sub(begin, i + 1));
next = i + 1;
}
else if (pos == 'S')
{
tokens.Add(sentence.Sub(i, i + 1));
next = i + 1;
}
}
if (next < sentence.Length)
{
tokens.Add(sentence.Substring(next));
}
return tokens;
}
#endregion
}
}

489
BotSharp.Core/Engines/Jieba.NET/JiebaSegmenter.cs Normal file
View file

@ -0,0 +1,489 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using JiebaNet.Segmenter.Common;
using JiebaNet.Segmenter.FinalSeg;
namespace JiebaNet.Segmenter
{
public class JiebaSegmenter
{
private static readonly WordDictionary WordDict = WordDictionary.Instance;
private static readonly IFinalSeg FinalSeg = Viterbi.Instance;
private static readonly ISet<string> LoadedPath = new HashSet<string>();
private static readonly object locker = new object();
internal IDictionary<string, string> UserWordTagTab { get; set; }
#region Regular Expressions
internal static readonly Regex RegexChineseDefault = new Regex(@"([\u4E00-\u9FD5a-zA-Z0-9+#&\._%]+)", RegexOptions.Compiled);
internal static readonly Regex RegexSkipDefault = new Regex(@"(\r\n|\s)", RegexOptions.Compiled);
internal static readonly Regex RegexChineseCutAll = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled);
internal static readonly Regex RegexSkipCutAll = new Regex(@"[^a-zA-Z0-9+#\n]", RegexOptions.Compiled);
internal static readonly Regex RegexEnglishChars = new Regex(@"[a-zA-Z0-9]", RegexOptions.Compiled);
internal static readonly Regex RegexUserDict = new Regex("^(?<word>.+?)(?<freq> [0-9]+)?(?<tag> [a-z]+)?$", RegexOptions.Compiled);
#endregion
public JiebaSegmenter()
{
UserWordTagTab = new Dictionary<string, string>();
}
/// <summary>
/// The main function that segments an entire sentence that contains
/// Chinese characters into seperated words.
/// </summary>
/// <param name="text">The string to be segmented.</param>
/// <param name="cutAll">Specify segmentation pattern. True for full pattern, False for accurate pattern.</param>
/// <param name="hmm">Whether to use the Hidden Markov Model.</param>
/// <returns></returns>
public IEnumerable<string> Cut(string text, bool cutAll = false, bool hmm = true)
{
var reHan = RegexChineseDefault;
var reSkip = RegexSkipDefault;
Func<string, IEnumerable<string>> cutMethod = null;
if (cutAll)
{
reHan = RegexChineseCutAll;
reSkip = RegexSkipCutAll;
}
if (cutAll)
{
cutMethod = CutAll;
}
else if (hmm)
{
cutMethod = CutDag;
}
else
{
cutMethod = CutDagWithoutHmm;
}
return CutIt(text, cutMethod, reHan, reSkip, cutAll);
}
public IEnumerable<string> CutForSearch(string text, bool hmm = true)
{
var result = new List<string>();
var words = Cut(text, hmm: hmm);
foreach (var w in words)
{
if (w.Length > 2)
{
foreach (var i in Enumerable.Range(0, w.Length - 1))
{
var gram2 = w.Substring(i, 2);
if (WordDict.ContainsWord(gram2))
{
result.Add(gram2);
}
}
}
if (w.Length > 3)
{
foreach (var i in Enumerable.Range(0, w.Length - 2))
{
var gram3 = w.Substring(i, 3);
if (WordDict.ContainsWord(gram3))
{
result.Add(gram3);
}
}
}
result.Add(w);
}
return result;
}
public IEnumerable<Token> Tokenize(string text, TokenizerMode mode = TokenizerMode.Default, bool hmm = true)
{
var result = new List<Token>();
var start = 0;
if (mode == TokenizerMode.Default)
{
foreach (var w in Cut(text, hmm: hmm))
{
var width = w.Length;
result.Add(new Token(w, start, start + width));
start += width;
}
}
else
{
foreach (var w in Cut(text, hmm: hmm))
{
var width = w.Length;
if (width > 2)
{
for (var i = 0; i < width - 1; i++)
{
var gram2 = w.Substring(i, 2);
if (WordDict.ContainsWord(gram2))
{
result.Add(new Token(gram2, start + i, start + i + 2));
}
}
}
if (width > 3)
{
for (var i = 0; i < width - 2; i++)
{
var gram3 = w.Substring(i, 3);
if (WordDict.ContainsWord(gram3))
{
result.Add(new Token(gram3, start + i, start + i + 3));
}
}
}
result.Add(new Token(w, start, start + width));
start += width;
}
}
return result;
}
#region Internal Cut Methods
internal IDictionary<int, List<int>> GetDag(string sentence)
{
var dag = new Dictionary<int, List<int>>();
var trie = WordDict.Trie;
var N = sentence.Length;
for (var k = 0; k < sentence.Length; k++)
{
var templist = new List<int>();
var i = k;
var frag = sentence.Substring(k, 1);
while (i < N && trie.ContainsKey(frag))
{
if (trie[frag] > 0)
{
templist.Add(i);
}
i++;
// TODO:
if (i < N)
{
frag = sentence.Sub(k, i + 1);
}
}
if (templist.Count == 0)
{
templist.Add(k);
}
dag[k] = templist;
}
return dag;
}
internal IDictionary<int, Pair<int>> Calc(string sentence, IDictionary<int, List<int>> dag)
{
var n = sentence.Length;
var route = new Dictionary<int, Pair<int>>();
route[n] = new Pair<int>(0, 0.0);
var logtotal = Math.Log(WordDict.Total);
for (var i = n - 1; i > -1; i--)
{
var candidate = new Pair<int>(-1, double.MinValue);
foreach (int x in dag[i])
{
var freq = Math.Log(WordDict.GetFreqOrDefault(sentence.Sub(i, x + 1))) - logtotal + route[x + 1].Freq;
if (candidate.Freq < freq)
{
candidate.Freq = freq;
candidate.Key = x;
}
}
route[i] = candidate;
}
return route;
}
internal IEnumerable<string> CutAll(string sentence)
{
var dag = GetDag(sentence);
var words = new List<string>();
var lastPos = -1;
foreach (var pair in dag)
{
var k = pair.Key;
var nexts = pair.Value;
if (nexts.Count == 1 && k > lastPos)
{
words.Add(sentence.Substring(k, nexts[0] + 1 - k));
lastPos = nexts[0];
}
else
{
foreach (var j in nexts)
{
if (j > k)
{
words.Add(sentence.Substring(k, j + 1 - k));
lastPos = j;
}
}
}
}
return words;
}
internal IEnumerable<string> CutDag(string sentence)
{
var dag = GetDag(sentence);
var route = Calc(sentence, dag);
var tokens = new List<string>();
var x = 0;
var n = sentence.Length;
var buf = string.Empty;
while (x < n)
{
var y = route[x].Key + 1;
var w = sentence.Substring(x, y - x);
if (y - x == 1)
{
buf += w;
}
else
{
if (buf.Length > 0)
{
AddBufferToWordList(tokens, buf);
buf = string.Empty;
}
tokens.Add(w);
}
x = y;
}
if (buf.Length > 0)
{
AddBufferToWordList(tokens, buf);
}
return tokens;
}
internal IEnumerable<string> CutDagWithoutHmm(string sentence)
{
var dag = GetDag(sentence);
var route = Calc(sentence, dag);
var words = new List<string>();
var x = 0;
string buf = string.Empty;
var N = sentence.Length;
var y = -1;
while (x < N)
{
y = route[x].Key + 1;
var l_word = sentence.Substring(x, y - x);
if (RegexEnglishChars.IsMatch(l_word) && l_word.Length == 1)
{
buf += l_word;
x = y;
}
else
{
if (buf.Length > 0)
{
words.Add(buf);
buf = string.Empty;
}
words.Add(l_word);
x = y;
}
}
if (buf.Length > 0)
{
words.Add(buf);
}
return words;
}
internal IEnumerable<string> CutIt(string text, Func<string, IEnumerable<string>> cutMethod,
Regex reHan, Regex reSkip, bool cutAll)
{
var result = new List<string>();
var blocks = reHan.Split(text);
foreach (var blk in blocks)
{
if (string.IsNullOrEmpty(blk))
{
continue;
}
if (reHan.IsMatch(blk))
{
foreach (var word in cutMethod(blk))
{
result.Add(word);
}
}
else
{
var tmp = reSkip.Split(blk);
foreach (var x in tmp)
{
if (reSkip.IsMatch(x))
{
result.Add(x);
}
else if (!cutAll)
{
foreach (var ch in x)
{
result.Add(ch.ToString());
}
}
else
{
result.Add(x);
}
}
}
}
return result;
}
#endregion
#region Extend Main Dict
/// <summary>
/// Loads user dictionaries.
/// </summary>
/// <param name="userDictFile"></param>
public void LoadUserDict(string userDictFile)
{
var dictFullPath = Path.GetFullPath(userDictFile);
Debug.WriteLine("Initializing user dictionary: " + userDictFile);
lock (locker)
{
if (LoadedPath.Contains(dictFullPath))
return;
try
{
var startTime = DateTime.Now.Millisecond;
var lines = File.ReadAllLines(dictFullPath, Encoding.UTF8);
foreach (var line in lines)
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var tokens = RegexUserDict.Match(line.Trim()).Groups;
var word = tokens["word"].Value.Trim();
var freq = tokens["freq"].Value.Trim();
var tag = tokens["tag"].Value.Trim();
var actualFreq = freq.Length > 0 ? int.Parse(freq) : 0;
AddWord(word, actualFreq, tag);
}
Debug.WriteLine("user dict '{0}' load finished, time elapsed {1} ms",
dictFullPath, DateTime.Now.Millisecond - startTime);
}
catch (IOException e)
{
Debug.Fail(string.Format("'{0}' load failure, reason: {1}", dictFullPath, e.Message));
}
catch (FormatException fe)
{
Debug.Fail(fe.Message);
}
}
}
public void AddWord(string word, int freq = 0, string tag = null)
{
if (freq <= 0)
{
freq = WordDict.SuggestFreq(word, Cut(word, hmm: false));
}
WordDict.AddWord(word, freq);
// Add user word tag of POS
if (!string.IsNullOrEmpty(tag))
{
UserWordTagTab[word] = tag;
}
}
public void DeleteWord(string word)
{
WordDict.DeleteWord(word);
}
#endregion
#region Private Helpers
private void AddBufferToWordList(List<string> words, string buf)
{
if (buf.Length == 1)
{
words.Add(buf);
}
else
{
if (!WordDict.ContainsWord(buf))
{
var tokens = FinalSeg.Cut(buf);
words.AddRange(tokens);
}
else
{
words.AddRange(buf.Select(ch => ch.ToString()));
}
}
}
#endregion
}
public enum TokenizerMode
{
Default,
Search
}
}

4
BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs
View file

@ -2,6 +2,7 @@
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Tag;
using JiebaNet.Segmenter;
using JiebaNet.Segmenter.PosSeg;
using Microsoft.Extensions.Configuration;
using System;
@ -39,6 +40,9 @@ namespace BotSharp.Core.Engines.Jieba.NET
{
if (posSeg == null)
{
string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
AppDomain.CurrentDomain.SetData("JiebaConfigFileDir", contentDir);
posSeg = new PosSegmenter();
}
}

7
BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs
View file

@ -34,10 +34,13 @@ namespace BotSharp.Core.Engines.Jieba.NET
private void Init()
{
if(segmenter == null)
if (segmenter == null)
{
string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
AppDomain.CurrentDomain.SetData("JiebaConfigFileDir", contentDir);
segmenter = new JiebaSegmenter();
segmenter.LoadUserDict($"App_Data{Path.DirectorySeparatorChar}userdict.txt");
segmenter.LoadUserDict(Path.Combine(contentDir, "userdict.txt"));
}
}
}

14
BotSharp.Core/Engines/Jieba.NET/Node.cs Normal file
View file

@ -0,0 +1,14 @@
namespace JiebaNet.Segmenter
{
public class Node
{
public char Value { get; private set; }
public Node Parent { get; private set; }
public Node(char value, Node parent)
{
Value = value;
Parent = parent;
}
}
}

19
BotSharp.Core/Engines/Jieba.NET/Pair.cs Normal file
View file

@ -0,0 +1,19 @@
namespace JiebaNet.Segmenter
{
public class Pair<TKey>
{
public TKey Key { get;set; }
public double Freq { get; set; }
public Pair(TKey key, double freq)
{
Key = key;
Freq = freq;
}
public override string ToString()
{
return "Candidate [Key=" + Key + ", Freq=" + Freq + "]";
}
}
}

18
BotSharp.Core/Engines/Jieba.NET/PosSeg/Pair.cs Normal file
View file

@ -0,0 +1,18 @@
namespace JiebaNet.Segmenter.PosSeg
{
public class Pair
{
public string Word { get; set; }
public string Flag { get; set; }
public Pair(string word, string flag)
{
Word = word;
Flag = flag;
}
public override string ToString()
{
return string.Format("{0}/{1}", Word, Flag);
}
}
}

301
BotSharp.Core/Engines/Jieba.NET/PosSeg/PosSegmenter.cs Normal file
View file

@ -0,0 +1,301 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using JiebaNet.Segmenter.Common;
namespace JiebaNet.Segmenter.PosSeg
{
public class PosSegmenter
{
private static readonly WordDictionary WordDict = WordDictionary.Instance;
private static readonly Viterbi PosSeg = Viterbi.Instance;
// TODO:
private static readonly object locker = new object();
#region Regular Expressions
internal static readonly Regex RegexChineseInternal = new Regex(@"([\u4E00-\u9FD5a-zA-Z0-9+#&\._]+)", RegexOptions.Compiled);
internal static readonly Regex RegexSkipInternal = new Regex(@"(\r\n|\s)", RegexOptions.Compiled);
internal static readonly Regex RegexChineseDetail = new Regex(@"([\u4E00-\u9FD5]+)", RegexOptions.Compiled);
internal static readonly Regex RegexSkipDetail = new Regex(@"([\.0-9]+|[a-zA-Z0-9]+)", RegexOptions.Compiled);
internal static readonly Regex RegexEnglishWords = new Regex(@"[a-zA-Z0-9]+", RegexOptions.Compiled);
internal static readonly Regex RegexNumbers = new Regex(@"[\.0-9]+", RegexOptions.Compiled);
internal static readonly Regex RegexEnglishChar = new Regex(@"^[a-zA-Z0-9]$", RegexOptions.Compiled);
#endregion
private static IDictionary<string, string> _wordTagTab;
static PosSegmenter()
{
LoadWordTagTab();
}
private static void LoadWordTagTab()
{
try
{
_wordTagTab = new Dictionary<string, string>();
var lines = FileExtension.ReadEmbeddedAllLines(ConfigManager.MainDictFile);
foreach (var line in lines)
{
var tokens = line.Split(' ');
if (tokens.Length < 2)
{
Debug.Fail(string.Format("Invalid line: {0}", line));
continue;
}
var word = tokens[0];
var tag = tokens[2];
_wordTagTab[word] = tag;
}
}
catch (System.IO.IOException e)
{
Debug.Fail(string.Format("Word tag table load failure, reason: {0}", e.Message));
}
catch (FormatException fe)
{
Debug.Fail(fe.Message);
}
}
private JiebaSegmenter _segmenter;
public PosSegmenter()
{
_segmenter = new JiebaSegmenter();
}
public PosSegmenter(JiebaSegmenter segmenter)
{
_segmenter = segmenter;
}
private void CheckNewUserWordTags()
{
if (_segmenter.UserWordTagTab.IsNotEmpty())
{
_wordTagTab.Update(_segmenter.UserWordTagTab);
_segmenter.UserWordTagTab = new Dictionary<string, string>();
}
}
public IEnumerable<Pair> Cut(string text, bool hmm = true)
{
return CutInternal(text, hmm);
}
#region Internal Cut Methods
internal IEnumerable<Pair> CutInternal(string text, bool hmm = true)
{
CheckNewUserWordTags();
var blocks = RegexChineseInternal.Split(text);
Func<string, IEnumerable<Pair>> cutMethod = null;
if (hmm)
{
cutMethod = CutDag;
}
else
{
cutMethod = CutDagWithoutHmm;
}
var tokens = new List<Pair>();
foreach (var blk in blocks)
{
if (RegexChineseInternal.IsMatch(blk))
{
tokens.AddRange(cutMethod(blk));
}
else
{
var tmp = RegexSkipInternal.Split(blk);
foreach (var x in tmp)
{
if (RegexSkipInternal.IsMatch(x))
{
tokens.Add(new Pair(x, "x"));
}
else
{
foreach (var xx in x)
{
// TODO: each char?
var xxs = xx.ToString();
if (RegexNumbers.IsMatch(xxs))
{
tokens.Add(new Pair(xxs, "m"));
}
else if (RegexEnglishWords.IsMatch(x))
{
tokens.Add(new Pair(xxs, "eng"));
}
else
{
tokens.Add(new Pair(xxs, "x"));
}
}
}
}
}
}
return tokens;
}
internal IEnumerable<Pair> CutDag(string sentence)
{
var dag = _segmenter.GetDag(sentence);
var route = _segmenter.Calc(sentence, dag);
var tokens = new List<Pair>();
var x = 0;
var n = sentence.Length;
var buf = string.Empty;
while (x < n)
{
var y = route[x].Key + 1;
var w = sentence.Substring(x, y - x);
if (y - x == 1)
{
buf += w;
}
else
{
if (buf.Length > 0)
{
AddBufferToWordList(tokens, buf);
buf = string.Empty;
}
tokens.Add(new Pair(w, _wordTagTab.GetDefault(w, "x")));
}
x = y;
}
if (buf.Length > 0)
{
AddBufferToWordList(tokens, buf);
}
return tokens;
}
internal IEnumerable<Pair> CutDagWithoutHmm(string sentence)
{
var dag = _segmenter.GetDag(sentence);
var route = _segmenter.Calc(sentence, dag);
var tokens = new List<Pair>();
var x = 0;
var buf = string.Empty;
var n = sentence.Length;
var y = -1;
while (x < n)
{
y = route[x].Key + 1;
var w = sentence.Substring(x, y - x);
// TODO: char or word?
if (RegexEnglishChar.IsMatch(w))
{
buf += w;
x = y;
}
else
{
if (buf.Length > 0)
{
tokens.Add(new Pair(buf, "eng"));
buf = string.Empty;
}
tokens.Add(new Pair(w, _wordTagTab.GetDefault(w, "x")));
x = y;
}
}
if (buf.Length > 0)
{
tokens.Add(new Pair(buf, "eng"));
}
return tokens;
}
internal IEnumerable<Pair> CutDetail(string text)
{
var tokens = new List<Pair>();
var blocks = RegexChineseDetail.Split(text);
foreach (var blk in blocks)
{
if (RegexChineseDetail.IsMatch(blk))
{
tokens.AddRange(PosSeg.Cut(blk));
}
else
{
var tmp = RegexSkipDetail.Split(blk);
foreach (var x in tmp)
{
if (!string.IsNullOrWhiteSpace(x))
{
if (RegexNumbers.IsMatch(x))
{
tokens.Add(new Pair(x, "m"));
}
else if(RegexEnglishWords.IsMatch(x))
{
tokens.Add(new Pair(x, "eng"));
}
else
{
tokens.Add(new Pair(x, "x"));
}
}
}
}
}
return tokens;
}
#endregion
#region Private Helpers
private void AddBufferToWordList(List<Pair> words, string buf)
{
if (buf.Length == 1)
{
words.Add(new Pair(buf, _wordTagTab.GetDefault(buf, "x")));
}
else
{
if (!WordDict.ContainsWord(buf))
{
var tokens = CutDetail(buf);
words.AddRange(tokens);
}
else
{
words.AddRange(buf.Select(ch => new Pair(ch.ToString(), "x")));
}
}
}
#endregion
}
}

176
BotSharp.Core/Engines/Jieba.NET/PosSeg/Viterbi.cs Normal file
View file

@ -0,0 +1,176 @@
using System;
using System.Collections.Generic;
using System.Linq;
using JiebaNet.Segmenter.Common;
using Newtonsoft.Json;
namespace JiebaNet.Segmenter.PosSeg
{
public class Viterbi
{
private static readonly Lazy<Viterbi> Lazy = new Lazy<Viterbi>(() => new Viterbi());
private static IDictionary<string, double> _startProbs;
private static IDictionary<string, IDictionary<string, double>> _transProbs;
private static IDictionary<string, IDictionary<char, double>> _emitProbs;
private static IDictionary<char, List<string>> _stateTab;
private Viterbi()
{
LoadModel();
}
// TODO: synchronized
public static Viterbi Instance
{
get { return Lazy.Value; }
}
public IEnumerable<Pair> Cut(string sentence)
{
var probPosList = ViterbiCut(sentence);
var posList = probPosList.Item2;
var tokens = new List<Pair>();
int begin = 0, next = 0;
for (var i = 0; i < sentence.Length; i++)
{
var parts = posList[i].Split('-');
var charState = parts[0][0];
var pos = parts[1];
if (charState == 'B')
begin = i;
else if (charState == 'E')
{
tokens.Add(new Pair(sentence.Sub(begin, i + 1), pos));
next = i + 1;
}
else if (charState == 'S')
{
tokens.Add(new Pair(sentence.Sub(i, i + 1), pos));
next = i + 1;
}
}
if (next < sentence.Length)
{
tokens.Add(new Pair(sentence.Substring(next), posList[next].Split('-')[1]));
}
return tokens;
}
#region Private Helpers
private static void LoadModel()
{
var startJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbStartFile);
_startProbs = JsonConvert.DeserializeObject<IDictionary<string, double>>(startJson);
var transJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbTransFile);
_transProbs = JsonConvert.DeserializeObject<IDictionary<string, IDictionary<string, double>>>(transJson);
var emitJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.PosProbEmitFile);
_emitProbs = JsonConvert.DeserializeObject<IDictionary<string, IDictionary<char, double>>>(emitJson);
var tabJson = FileExtension.ReadEmbeddedAllLine(ConfigManager.CharStateTabFile);
_stateTab = JsonConvert.DeserializeObject<IDictionary<char, List<string>>>(tabJson);
}
// TODO: change sentence to obs?
private Tuple<double, List<string>> ViterbiCut(string sentence)
{
var v = new List<IDictionary<string, double>>();
var memPath = new List<IDictionary<string, string>>();
var allStates = _transProbs.Keys.ToList();
// Init weights and paths.
v.Add(new Dictionary<string, Double>());
memPath.Add(new Dictionary<string, string>());
foreach (var state in _stateTab.GetDefault(sentence[0], allStates))
{
var emP = _emitProbs[state].GetDefault(sentence[0], Constants.MinProb);
v[0][state] = _startProbs[state] + emP;
memPath[0][state] = string.Empty;
}
// For each remaining char
for (var i = 1; i < sentence.Length; ++i)
{
v.Add(new Dictionary<string, double>());
memPath.Add(new Dictionary<string, string>());
var prevStates = memPath[i - 1].Keys.Where(k => _transProbs[k].Count > 0);
var curPossibleStates = new HashSet<string>(prevStates.SelectMany(s => _transProbs[s].Keys));
IEnumerable<string> obsStates = _stateTab.GetDefault(sentence[i], allStates);
obsStates = curPossibleStates.Intersect(obsStates);
if (!obsStates.Any())
{
if (curPossibleStates.Count > 0)
{
obsStates = curPossibleStates;
}
else
{
obsStates = allStates;
}
}
foreach (var y in obsStates)
{
var emp = _emitProbs[y].GetDefault(sentence[i], Constants.MinProb);
var prob = double.MinValue;
var state = string.Empty;
foreach (var y0 in prevStates)
{
var tranp = _transProbs[y0].GetDefault(y, double.MinValue);
tranp = v[i - 1][y0] + tranp + emp;
// TODO: compare two very small values;
// TODO: how to deal with negative infinity
if (prob < tranp ||
(prob == tranp && string.Compare(state, y0, StringComparison.CurrentCultureIgnoreCase) < 0))
{
prob = tranp;
state = y0;
}
}
v[i][y] = prob;
memPath[i][y] = state;
}
}
var vLast = v.Last();
var last = memPath.Last().Keys.Select(y => new {State = y, Prob = vLast[y]});
var endProb = double.MinValue;
var endState = string.Empty;
foreach (var endPoint in last)
{
// TODO: compare two very small values;
if (endProb < endPoint.Prob ||
(endProb == endPoint.Prob && String.Compare(endState, endPoint.State, StringComparison.CurrentCultureIgnoreCase) < 0))
{
endProb = endPoint.Prob;
endState = endPoint.State;
}
}
var route = new string[sentence.Length];
var n = sentence.Length - 1;
var curState = endState;
while(n >= 0)
{
route[n] = curState;
curState = memPath[n][curState];
n--;
}
return new Tuple<double, List<string>>(endProb, route.ToList());
}
#endregion
}
}

158
BotSharp.Core/Engines/Jieba.NET/Spelling/SpellChecker.cs Normal file
View file

@ -0,0 +1,158 @@
using System.Collections.Generic;
using System.Linq;
using JiebaNet.Segmenter.Common;
namespace JiebaNet.Segmenter.Spelling
{
public interface ISpellChecker
{
IEnumerable<string> Suggests(string word);
}
public class SpellChecker : ISpellChecker
{
internal static readonly WordDictionary WordDict = WordDictionary.Instance;
internal readonly Trie WordTrie;
internal readonly Dictionary<char, HashSet<char>> FirstChars;
public SpellChecker()
{
var wordDict = WordDictionary.Instance;
WordTrie = new Trie();
FirstChars = new Dictionary<char, HashSet<char>>();
foreach (var wd in wordDict.Trie)
{
if (wd.Value > 0)
{
WordTrie.Insert(wd.Key, wd.Value);
if (wd.Key.Length >= 2)
{
var second = wd.Key[1];
var first = wd.Key[0];
if (!FirstChars.ContainsKey(second))
{
FirstChars[second] = new HashSet<char>();
}
FirstChars[second].Add(first);
}
}
}
}
internal ISet<string> GetEdits1(string word)
{
var splits = new List<WordSplit>();
for (var i = 0; i <= word.Length; i++)
{
splits.Add(new WordSplit() { Left = word.Substring(0, i), Right = word.Substring(i) });
}
var deletes = splits
.Where(s => !string.IsNullOrEmpty(s.Right))
.Select(s => s.Left + s.Right.Substring(1));
var transposes = splits
.Where(s => s.Right.Length > 1)
.Select(s => s.Left + s.Right[1] + s.Right[0] + s.Right.Substring(2));
var replaces = new HashSet<string>();
if (word.Length > 1)
{
var firsts = FirstChars[word[1]];
foreach (var first in firsts)
{
if (first != word[0])
{
replaces.Add(first + word.Substring(1));
}
}
var node = WordTrie.Root.Children[word[0]];
for (int i = 1; node.IsNotNull() && node.Children.IsNotEmpty() && i < word.Length; i++)
{
foreach (var c in node.Children.Keys)
{
replaces.Add(word.Substring(0, i) + c + word.Substring(i + 1));
}
node = node.Children.GetValueOrDefault(word[i]);
}
}
var inserts = new HashSet<string>();
if (word.Length > 1)
{
if (FirstChars.ContainsKey(word[0]))
{
var firsts = FirstChars[word[0]];
foreach (var first in firsts)
{
inserts.Add(first + word);
}
}
var node = WordTrie.Root.Children.GetValueOrDefault(word[0]);
for (int i = 0; node.IsNotNull() && node.Children.IsNotEmpty() && i < word.Length; i++)
{
foreach (var c in node.Children.Keys)
{
inserts.Add(word.Substring(0, i+1) + c + word.Substring(i+1));
}
if (i < word.Length - 1)
{
node = node.Children.GetValueOrDefault(word[i + 1]);
}
}
}
var result = new HashSet<string>();
result.UnionWith(deletes);
result.UnionWith(transposes);
result.UnionWith(replaces);
result.UnionWith(inserts);
return result;
}
internal ISet<string> GetKnownEdits2(string word)
{
var result = new HashSet<string>();
foreach (var e1 in GetEdits1(word))
{
result.UnionWith(GetEdits1(e1).Where(e => WordDictionary.Instance.ContainsWord(e)));
}
return result;
}
internal ISet<string> GetKnownWords(IEnumerable<string> words)
{
return new HashSet<string>(words.Where(w => WordDictionary.Instance.ContainsWord(w)));
}
public IEnumerable<string> Suggests(string word)
{
if (WordDict.ContainsWord(word))
{
return new[] {word};
}
var candicates = GetKnownWords(GetEdits1(word));
if (candicates.IsNotEmpty())
{
return candicates.OrderByDescending(c => WordDict.GetFreqOrDefault(c));
}
candicates.UnionWith(GetKnownEdits2(word));
return candicates.OrderByDescending(c => WordDict.GetFreqOrDefault(c));
}
}
internal class WordSplit
{
public string Left { get; set; }
public string Right { get; set; }
}
}

21
BotSharp.Core/Engines/Jieba.NET/Token.cs Normal file
View file

@ -0,0 +1,21 @@
namespace JiebaNet.Segmenter
{
public class Token
{
public string Word { get; set; }
public int StartIndex { get; set; }
public int EndIndex { get; set; }
public Token(string word, int startIndex, int endIndex)
{
Word = word;
StartIndex = startIndex;
EndIndex = endIndex;
}
public override string ToString()
{
return string.Format("[{0}, ({1}, {2})]", Word, StartIndex, EndIndex);
}
}
}

136
BotSharp.Core/Engines/Jieba.NET/WordDictionary.cs Normal file
View file

@ -0,0 +1,136 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using JiebaNet.Segmenter.Common;
using Microsoft.Extensions.FileProviders;
using System.Reflection;
namespace JiebaNet.Segmenter
{
public class WordDictionary
{
private static readonly Lazy<WordDictionary> lazy = new Lazy<WordDictionary>(() => new WordDictionary());
private static readonly string MainDict = ConfigManager.MainDictFile;
internal IDictionary<string, int> Trie = new Dictionary<string, int>();
/// <summary>
/// total occurrence of all words.
/// </summary>
public double Total { get; set; }
private WordDictionary()
{
LoadDict();
Debug.WriteLine("{0} words (and their prefixes)", Trie.Count);
Debug.WriteLine("total freq: {0}", Total);
}
public static WordDictionary Instance
{
get { return lazy.Value; }
}
private void LoadDict()
{
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var filePath = ConfigManager.MainDictFile;
using (var sr = new StreamReader(filePath))
{
string line = null;
while ((line = sr.ReadLine()) != null)
{
var tokens = line.Split(' ');
if (tokens.Length < 2)
{
Debug.Fail(string.Format("Invalid line: {0}", line));
continue;
}
var word = tokens[0];
var freq = int.Parse(tokens[1]);
Trie[word] = freq;
Total += freq;
foreach (var ch in Enumerable.Range(0, word.Length))
{
var wfrag = word.Sub(0, ch + 1);
if (!Trie.ContainsKey(wfrag))
{
Trie[wfrag] = 0;
}
}
}
}
stopWatch.Stop();
Debug.WriteLine("main dict load finished, time elapsed {0} ms", stopWatch.ElapsedMilliseconds);
}
catch (IOException e)
{
Debug.Fail(string.Format("{0} load failure, reason: {1}", MainDict, e.Message));
}
catch (FormatException fe)
{
Debug.Fail(fe.Message);
}
}
public bool ContainsWord(string word)
{
return Trie.ContainsKey(word) && Trie[word] > 0;
}
public int GetFreqOrDefault(string key)
{
if (ContainsWord(key))
return Trie[key];
else
return 1;
}
public void AddWord(string word, int freq, string tag = null)
{
if (ContainsWord(word))
{
Total -= Trie[word];
}
Trie[word] = freq;
Total += freq;
for (var i = 0; i < word.Length; i++)
{
var wfrag = word.Substring(0, i + 1);
if (!Trie.ContainsKey(wfrag))
{
Trie[wfrag] = 0;
}
}
}
public void DeleteWord(string word)
{
AddWord(word, 0);
}
internal int SuggestFreq(string word, IEnumerable<string> segments)
{
double freq = 1;
foreach (var seg in segments)
{
freq *= GetFreqOrDefault(seg) / Total;
}
return Math.Max((int)(freq * Total) + 1, GetFreqOrDefault(word));
}
}
}

14
BotSharp.NLP/BotSharp.NLP.csproj
View file

@ -31,15 +31,11 @@ Naive Bayes Classifier</Description>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
<DefineConstants>DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA NLU|AnyCPU'">
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DefineConstants>RASA;DEBUG;TRACE</DefineConstants>
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BotSharp.Algorithm" Version="0.1.0" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
</ItemGroup>
@ -47,10 +43,4 @@ Naive Bayes Classifier</Description>
<ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
</ItemGroup>
<ItemGroup>
<Compile Update="NER\README.cs">
<SubType>Code</SubType>
</Compile>
</ItemGroup>
</Project>

1
BotSharp.RestApi/BotSharp.RestApi.csproj
View file

@ -27,6 +27,7 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
<DocumentationFile>BotSharp.RestApi.xml</DocumentationFile>
<DefineConstants>TRACE;DEBUG</DefineConstants>
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA NLU|AnyCPU'">

5
BotSharp.Vision/BotSharp.Vision.csproj
View file

@ -2,6 +2,11 @@
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Configurations>Debug;Release;RASA;DIALOGFLOW</Configurations>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
</Project>

5
BotSharp.Voice/BotSharp.Voice.csproj
View file

@ -2,6 +2,11 @@
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Configurations>Debug;Release;RASA;DIALOGFLOW</Configurations>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
</Project>

3
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -12,7 +12,8 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
<DefineConstants>TRACE</DefineConstants>
<DefineConstants>DEBUG;TRACE;RASA</DefineConstants>
<OutputPath>bin\RASA</OutputPath>
</PropertyGroup>
<ItemGroup>

22
BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml
View file

@ -1,22 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
This file is used by the publish/package process of your Web project. You can customize the behavior of this process
by editing this MSBuild file. In order to learn more about this please visit https://go.microsoft.com/fwlink/?LinkID=208121.
-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<WebPublishMethod>FileSystem</WebPublishMethod>
<PublishProvider>FileSystem</PublishProvider>
<LastUsedBuildConfiguration>RASA</LastUsedBuildConfiguration>
<LastUsedPlatform>Any CPU</LastUsedPlatform>
<SiteUrlToLaunchAfterPublish />
<LaunchSiteAfterPublish>True</LaunchSiteAfterPublish>
<ExcludeApp_Data>False</ExcludeApp_Data>
<TargetFramework>netcoreapp2.1</TargetFramework>
<ProjectGuid>03dca427-327a-4fc9-9a2f-57d17f16708c</ProjectGuid>
<SelfContained>false</SelfContained>
<_IsPortable>true</_IsPortable>
<publishUrl>PublishOutput</publishUrl>
<DeleteExistingFiles>True</DeleteExistingFiles>
</PropertyGroup>
</Project>

11
BotSharp.WebHost/Properties/PublishProfiles/RASA.pubxml.user
View file

@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
This file is used by the publish/package process of your Web project. You can customize the behavior of this process
by editing this MSBuild file. In order to learn more about this please visit https://go.microsoft.com/fwlink/?LinkID=208121.
-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<TimeStampOfAssociatedLegacyPublishXmlFile />
<_PublishTargetUrl>D:\Projects\BotSharp\BotSharp.WebHost\PublishOutput</_PublishTargetUrl>
</PropertyGroup>
</Project>

52
BotSharp.sln
View file

@ -8,17 +8,17 @@ EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.RestApi", "BotSharp.RestApi\BotSharp.RestApi.csproj", "{80DDAA05-69BC-49DD-96A4-37345E7A2E20}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotSharp.WebHost\BotSharp.WebHost.csproj", "{03DCA427-327A-4FC9-9A2F-57D17F16708C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.UnitTest", "BotSharp.Core.UnitTest\BotSharp.Core.UnitTest.csproj", "{A31A6853-DFB8-477D-8F09-8E6E3D166102}"
ProjectSection(ProjectDependencies) = postProject
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF} = {BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22} = {C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}
{75B02A7C-EDB3-4082-9C1F-471773E760C3} = {75B02A7C-EDB3-4082-9C1F-471773E760C3}
{184F8B93-68C2-4849-9625-4023C4360990} = {184F8B93-68C2-4849-9625-4023C4360990}
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Algorithm", "BotSharp.Algorithm\BotSharp.Algorithm.csproj", "{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Algorithm.UnitTest", "BotSharp.Algorithm.UnitTest\BotSharp.Algorithm.UnitTest.csproj", "{4C4EE7A8-99CA-41FE-9517-C66799720784}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NLP\BotSharp.NLP.csproj", "{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Voice", "BotSharp.Voice\BotSharp.Voice.csproj", "{184F8B93-68C2-4849-9625-4023C4360990}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Vision", "BotSharp.Vision\BotSharp.Vision.csproj", "{75B02A7C-EDB3-4082-9C1F-471773E760C3}"
@ -55,14 +55,6 @@ Global
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.RASA|Any CPU.Build.0 = RASA|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.Build.0 = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.RASA|Any CPU.ActiveCfg = RASA|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.RASA|Any CPU.Build.0 = RASA|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.Build.0 = Release|Any CPU
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
@ -71,14 +63,6 @@ Global
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.RASA|Any CPU.Build.0 = RASA|Any CPU
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BE1A033F-AC14-4654-95C3-4B33AF8F6BBF}.Release|Any CPU.Build.0 = Release|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.RASA|Any CPU.ActiveCfg = RASA|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.RASA|Any CPU.Build.0 = RASA|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4C4EE7A8-99CA-41FE-9517-C66799720784}.Release|Any CPU.Build.0 = Release|Any CPU
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
@ -87,28 +71,20 @@ Global
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.RASA|Any CPU.Build.0 = RASA|Any CPU
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C2FDC855-BD88-4041-B0FF-3AA8A1C11A22}.Release|Any CPU.Build.0 = Release|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.RASA|Any CPU.ActiveCfg = RASA|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.RASA|Any CPU.Build.0 = RASA|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}.Release|Any CPU.Build.0 = Release|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.Debug|Any CPU.Build.0 = Debug|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.ActiveCfg = Debug|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.Build.0 = Debug|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.ActiveCfg = Release|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.Build.0 = Release|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.ActiveCfg = RASA|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.RASA|Any CPU.Build.0 = RASA|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.Release|Any CPU.ActiveCfg = Release|Any CPU
{184F8B93-68C2-4849-9625-4023C4360990}.Release|Any CPU.Build.0 = Release|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.ActiveCfg = Debug|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.Build.0 = Debug|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.ActiveCfg = Release|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.Build.0 = Release|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.ActiveCfg = DIALOGFLOW|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.DIALOGFLOW|Any CPU.Build.0 = DIALOGFLOW|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.ActiveCfg = RASA|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.RASA|Any CPU.Build.0 = RASA|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{75B02A7C-EDB3-4082-9C1F-471773E760C3}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection

8
Dockerfile
View file

@ -4,17 +4,13 @@ WORKDIR /source
# copies the rest of your code
COPY . .
RUN dotnet build
RUN dotnet publish /p:PublishProfile=RASA /p:Configuration=RASA --output /app
# RUN dotnet build
RUN dotnet publish BotSharp.WebHost/BotSharp.WebHost.csproj --configuration RASA --output /app
# copy Settings folder
WORKDIR /app
COPY Settings Settings
RUN mkdir App_Data/Projects
# move data for jieba.NetCore
# RUN mv App_Data/Resources Resources
# RUN mv App_Data/userdict.txt userdict.txt
# stage 2: run
ENTRYPOINT [ "dotnet", "BotSharp.WebHost.dll" ]

8
README.rst
View file

@ -4,7 +4,7 @@
The Open Source AI Bot Platform Builder
======================================================
.. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg
:target: `gitter`_
:alt: gitter
@ -18,8 +18,10 @@ The Open Source AI Bot Platform Builder
.. image:: https://img.shields.io/nuget/dt/EntityFrameworkCore.BootKit.svg
:target: `botsharpnuget`_
:alt: NuGet
This project is for learning purposes only, please do not use it in a production environment.
**********************************************************************************************
*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our .NET developers using the BotSharp AI BOT Platform Builder to build a CaaP. It opens up as much learning power as possible for your own robots and precisely control every step of the AI processing pipeline."*
**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.