SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

CRFLite encoder and decoder.

Browse source
This commit is contained in:
Oceania2018 2018-08-18 22:42:51 -05:00
parent db0c75a04b
commit 1a7c8ce612
16 changed files with 157 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -299,3 +299,4 @@ __pycache__/
/BotSharp.WebHost/App_Data/PredictFiles
/BotSharp.WebHost/App_Data/Projects
/BotSharp.WebHost/PublishOutput
/Data

2
BotSharp.UnitTest/AgentTest.cs → BotSharp.Core.UnitTest/AgentTest.cs
View file

@ -13,7 +13,7 @@ using System.IO;
using System.Linq;
using System.Text;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
[TestClass]
public class AgentTest : TestEssential

0
BotSharp.UnitTest/BotSharp.UnitTest.csproj → BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj
View file

2
BotSharp.UnitTest/BotTrainerTest.cs → BotSharp.Core.UnitTest/BotTrainerTest.cs
View file

@ -6,7 +6,7 @@ using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
[TestClass]
public class BotTrainerTest : TestEssential

2
BotSharp.UnitTest/ConversationTest.cs → BotSharp.Core.UnitTest/ConversationTest.cs
View file

@ -8,7 +8,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
[TestClass]
public class ConversationTest : TestEssential

2
BotSharp.UnitTest/EntityEntryTest.cs → BotSharp.Core.UnitTest/EntityEntryTest.cs
View file

@ -3,7 +3,7 @@ using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
[TestClass]
public class EntityEntryTest : TestEssential

2
BotSharp.UnitTest/EntityTypeTest.cs → BotSharp.Core.UnitTest/EntityTypeTest.cs
View file

@ -3,7 +3,7 @@ using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
[TestClass]
public class EntityTypeTest : TestEssential

2
BotSharp.UnitTest/TestEssential.cs → BotSharp.Core.UnitTest/TestEssential.cs
View file

@ -6,7 +6,7 @@ using System;
using System.IO;
using System.Linq;
namespace BotSharp.UnitTest
namespace BotSharp.Core.UnitTest
{
public abstract class TestEssential
{

6
BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj
View file

@ -6,6 +6,12 @@
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Data\**" />
<EmbeddedResource Remove="Data\**" />
<None Remove="Data\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.7.0" />
<PackageReference Include="MSTest.TestAdapter" Version="1.2.1" />

43
BotSharp.MachineLearning.UnitTest/DecoderTest.cs → BotSharp.MachineLearning.UnitTest/CRFLite/DecoderTest.cs
View file

@ -8,7 +8,7 @@ using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.MachineLearning.UnitTest
namespace BotSharp.MachineLearning.UnitTest.CRFLite
{
[TestClass]
public class DecoderTest
@ -17,9 +17,11 @@ namespace BotSharp.MachineLearning.UnitTest
public void TestDecode()
{
var encoder = new CRFDecoder();
bool bRet = Decode(new DecoderOptions
bool result = Decode(new DecoderOptions
{
InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\test.txt",
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng",
OutputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\output.txt"
});
}
@ -29,28 +31,17 @@ namespace BotSharp.MachineLearning.UnitTest
{
var parallelOption = new ParallelOptions();
var watch = Stopwatch.StartNew();
if (File.Exists(options.strInputFileName) == false)
{
//Logger.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName);
return false;
}
if (File.Exists(options.strModelFileName) == false)
{
//Logger.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName);
return false;
}
var sr = new StreamReader(options.strInputFileName);
var sr = new StreamReader(options.InputFileName);
StreamWriter sw = null, swSeg = null;
if (options.strOutputFileName != null && options.strOutputFileName.Length > 0)
if (options.OutputFileName != null && options.OutputFileName.Length > 0)
{
sw = new StreamWriter(options.strOutputFileName);
sw = new StreamWriter(options.OutputFileName);
}
if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0)
if (options.OutputSegFileName != null && options.OutputSegFileName.Length > 0)
{
swSeg = new StreamWriter(options.strOutputSegFileName);
swSeg = new StreamWriter(options.OutputSegFileName);
}
//Create CRFSharp wrapper instance. It's a global instance
@ -58,22 +49,22 @@ namespace BotSharp.MachineLearning.UnitTest
//Load encoded model from file
//Logger.WriteLine("Loading model from {0}", options.strModelFileName);
crfWrapper.LoadModel(options.strModelFileName);
crfWrapper.LoadModel(options.ModelFileName);
var queueRecords = new ConcurrentQueue<List<List<string>>>();
var queueSegRecords = new ConcurrentQueue<List<List<string>>>();
parallelOption.MaxDegreeOfParallelism = options.thread;
Parallel.For(0, options.thread, parallelOption, t =>
parallelOption.MaxDegreeOfParallelism = options.Thread;
Parallel.For(0, options.Thread, parallelOption, t =>
{
//Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
var tagger = crfWrapper.CreateTagger(options.nBest, options.maxword);
tagger.set_vlevel(options.probLevel);
var tagger = crfWrapper.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new crf_seg_out[options.nBest];
for (var i = 0; i < options.nBest; i++)
var crf_out = new crf_seg_out[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
}

22
BotSharp.MachineLearning.UnitTest/CRFLite/EncoderTest.cs Normal file
View file

@ -0,0 +1,22 @@
using BotSharp.MachineLearning.CRFLite;
using BotSharp.MachineLearning.CRFLite.Encoder;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace BotSharp.MachineLearning.UnitTest.CRFLite
{
[TestClass]
public class EncoderTest
{
[TestMethod]
public void TestEncode()
{
var encoder = new CRFEncoder();
bool result = encoder.Learn(new EncoderOptions
{
TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\corpus\eng.1K.training",
TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\template.NE",
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng"
});
}
}
}

22
BotSharp.MachineLearning.UnitTest/EncoderTest.cs
View file

@ -1,22 +0,0 @@
using BotSharp.MachineLearning.CRFLite;
using BotSharp.MachineLearning.CRFLite.Encoder;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace BotSharp.MachineLearning.UnitTest
{
[TestClass]
public class EncoderTest
{
[TestMethod]
public void TestEncode()
{
var encoder = new CRFEncoder();
bool bRet = encoder.Learn(new EncoderOptions
{
TrainingCorpusFileName = "",
TemplateFileName = "",
ModelFileName = ""
});
}
}
}

55
BotSharp.MachineLearning/CRFLite/Decoder/DecoderOptions.cs
View file

@ -8,23 +8,54 @@ namespace BotSharp.MachineLearning.CRFLite.Decoder
{
public class DecoderOptions
{
/// <summary>
///
/// </summary>
[Required]
public string strModelFileName;
public string ModelFileName;
/// <summary>
///
/// </summary>
[Required]
public string strInputFileName;
public string strOutputFileName;
public string strOutputSegFileName;
public int nBest;
public int thread;
public int probLevel;
public int maxword;
public string InputFileName;
/// <summary>
///
/// </summary>
public string OutputFileName;
/// <summary>
///
/// </summary>
public string OutputSegFileName;
/// <summary>
///
/// </summary>
public int NBest;
/// <summary>
///
/// </summary>
public int Thread;
/// <summary>
///
/// </summary>
public int ProbLevel;
/// <summary>
///
/// </summary>
public int MaxWord;
public DecoderOptions()
{
thread = 1;
nBest = 1;
probLevel = 0;
maxword = 100;
Thread = 1;
NBest = 1;
ProbLevel = 0;
MaxWord = 100;
}
}
}

2
BotSharp.MachineLearning/CRFLite/Encoder/EncoderOptions.cs
View file

@ -68,7 +68,7 @@ namespace BotSharp.MachineLearning.CRFLite.Encoder
/// <summary>
///
/// </summary>
public uint HugeLexMemLoad = 0;
public uint HugeLexMemLoad { get; set; }
/// <summary>
/// cost factor, too big or small value may lead encoded model over tune or under tune

48
BotSharp.sln
View file

@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.27130.2010
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.UnitTest", "BotSharp.UnitTest\BotSharp.UnitTest.csproj", "{1C883AA3-9182-469A-A17E-697DCAAB0121}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core", "BotSharp.Core\BotSharp.Core.csproj", "{95780673-2A1A-4953-962F-C46CBFDD07FF}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.RestApi", "BotSharp.RestApi\BotSharp.RestApi.csproj", "{80DDAA05-69BC-49DD-96A4-37345E7A2E20}"
@ -17,46 +15,82 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NL
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.MachineLearning.UnitTest", "BotSharp.MachineLearning.UnitTest\BotSharp.MachineLearning.UnitTest.csproj", "{B876F0E9-40F0-48B7-91D5-E09E0B442266}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.MachineLearning.UnitTest", "BotSharp.MachineLearning.UnitTest\BotSharp.MachineLearning.UnitTest.csproj", "{B876F0E9-40F0-48B7-91D5-E09E0B442266}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.UnitTest", "BotSharp.Core.UnitTest\BotSharp.Core.UnitTest.csproj", "{A31A6853-DFB8-477D-8F09-8E6E3D166102}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{1C883AA3-9182-469A-A17E-697DCAAB0121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1C883AA3-9182-469A-A17E-697DCAAB0121}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1C883AA3-9182-469A-A17E-697DCAAB0121}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1C883AA3-9182-469A-A17E-697DCAAB0121}.Release|Any CPU.Build.0 = Release|Any CPU
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Debug|x64.ActiveCfg = Debug|x64
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Debug|x64.Build.0 = Debug|x64
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Release|Any CPU.Build.0 = Release|Any CPU
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Release|x64.ActiveCfg = Release|x64
{95780673-2A1A-4953-962F-C46CBFDD07FF}.Release|x64.Build.0 = Release|x64
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Debug|Any CPU.Build.0 = Debug|Any CPU
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Debug|x64.ActiveCfg = Debug|x64
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Debug|x64.Build.0 = Debug|x64
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Release|Any CPU.ActiveCfg = Release|Any CPU
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Release|Any CPU.Build.0 = Release|Any CPU
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Release|x64.ActiveCfg = Release|x64
{80DDAA05-69BC-49DD-96A4-37345E7A2E20}.Release|x64.Build.0 = Release|x64
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Debug|x64.ActiveCfg = Debug|x64
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Debug|x64.Build.0 = Debug|x64
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|Any CPU.Build.0 = Release|Any CPU
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|x64.ActiveCfg = Release|x64
{03DCA427-327A-4FC9-9A2F-57D17F16708C}.Release|x64.Build.0 = Release|x64
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|x64.ActiveCfg = Debug|x64
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|x64.Build.0 = Debug|x64
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.Build.0 = Release|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|x64.ActiveCfg = Release|x64
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|x64.Build.0 = Release|x64
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|x64.ActiveCfg = Debug|x64
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|x64.Build.0 = Debug|x64
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|Any CPU.Build.0 = Release|Any CPU
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|x64.ActiveCfg = Release|x64
{D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|x64.Build.0 = Release|x64
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|x64.ActiveCfg = Debug|x64
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|x64.Build.0 = Debug|x64
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.Build.0 = Release|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|x64.ActiveCfg = Release|x64
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|x64.Build.0 = Release|x64
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|x64.ActiveCfg = Debug|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|x64.Build.0 = Debug|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.Build.0 = Release|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|x64.ActiveCfg = Release|Any CPU
{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|x64.Build.0 = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|x64.ActiveCfg = Debug|x64
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Debug|x64.Build.0 = Debug|x64
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.Build.0 = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.ActiveCfg = Release|x64
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

23
README.md
View file

@ -1,4 +1,4 @@
# <img src="https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png" height="30">
# <img src="https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png" height="30">
### The Open Source AI Bot Platform Builder for Enterprise
#### Open up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline.
**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.
@ -22,22 +22,39 @@ BotSharp is in accordance with components princple strictly, decouples every par
https://github.com/Oceania2018/BotSharp/wiki
## QUICK START
### Building BotSharp
Make sure the [Microsoft .NET Core](https://www.microsoft.com/net/download) build environment is installed.
Building solution using dotnet CLI (preferred).
```
PS D:\> git clone https://github.com/Oceania2018/BotSharp
PS D:\> cd BotSharp
PS D:\> dotnet build
```
### Install in docker container
Make sure you've got [Docker](https://www.docker.com/) installed:
```
PS D:\> git clone https://github.com/Oceania2018/BotSharp
PS D:\> cd BotSharp
```
Build docker image:
```
PS D:\BotSharp\> docker build -t botsharp .
```
Start a container:
```
PS D:\BotSharp\> docker run -it -p 3112:3112 botsharp
```
### Install in NuGet
````shell
```
PM> Install-Package BotSharp.Core
PM> Install-Package BotSharp.RestApi
````
```
Use BotSharp.NLP as a natural language processing toolkit alone。
```
PM> Install-Package BotSharp.NLP
```
#### Tip Jar