diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 7de0110f..b6c6c994 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -48,6 +48,7 @@ namespace BotSharp.Core.Engines provider.Configuration = Database.Configuration.GetSection("BotSharpAi"); provider.Process(agent, data); + //var corpus = agent.GrabCorpus(dc); // pipe process var pipelines = Database.Configuration.GetSection($"{config}:Pipe").Value diff --git a/BotSharp.Core/Engines/CRFsuite/Attribute.cs b/BotSharp.Core/Engines/CRFsuite/Attribute.cs new file mode 100644 index 00000000..be922b97 --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/Attribute.cs @@ -0,0 +1,78 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class Attribute : global::System.IDisposable { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal Attribute(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(Attribute obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~Attribute() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_Attribute(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public string attr { + set { + crfsuitePINVOKE.Attribute_attr_set(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + get { + string ret = crfsuitePINVOKE.Attribute_attr_get(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + } + + public double value { + set { + crfsuitePINVOKE.Attribute_value_set(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + get { + double ret = crfsuitePINVOKE.Attribute_value_get(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + } + + public Attribute() : this(crfsuitePINVOKE.new_Attribute__SWIG_0(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Attribute(string name) : this(crfsuitePINVOKE.new_Attribute__SWIG_1(name), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Attribute(string name, double val) : this(crfsuitePINVOKE.new_Attribute__SWIG_2(name, val), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs new file mode 100644 index 00000000..7bddad3b --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -0,0 +1,272 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using EntityFrameworkCore.BootKit; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.Core.Engines.CRFsuite +{ + public class CRFsuiteEntityRecognizer : INlpPipeline + { + public IConfiguration Configuration { get; set; } + + public bool Process(Agent agent, JObject data) + { + var dc = new DefaultDataContextLoader().GetDefaultDc(); + var corpus = agent.GrabCorpus(dc); + + // Mock Data + List train_sent = new List(); + train_sent.Add(new TrainingData("Melbourne", "NP", "B-LOC")); + train_sent.Add(new TrainingData("(", "Fpa", "O")); + train_sent.Add(new TrainingData("Australia", "NP", "B-LOC")); + train_sent.Add(new TrainingData(")", "Fpt", "O")); + train_sent.Add(new TrainingData(",", "Fc", "O")); + train_sent.Add(new TrainingData("25", "Z", "O")); + train_sent.Add(new TrainingData("may", "NC", "O")); + train_sent.Add(new TrainingData("(", "Fpa", "O")); + train_sent.Add(new TrainingData("EFE", "NC", "B-ORG")); + train_sent.Add(new TrainingData(")", "Fpt", "O")); + train_sent.Add(new TrainingData(".", "Fp", "O")); + + List> train_sents = new List>(); + train_sents.Add(train_sent); + + List X_train = new List(); + train_sents.ForEach(cur_sent => X_train.Add(new ItemSequence(sent2features(cur_sent)))); + + StringList sl = new StringList(); + + + List y_train = new List(); + train_sents.ForEach(cur_sent => y_train.Add(new StringList(sent2labels(cur_sent)))); + + Fit(X_train,y_train); + + + + + + return true; + } + /* + public List Merge(List sentence, List entities ) + { + List trainingTuple = new List(); + + HashSet entityWordBag = new HashSet(); + + entities.ForEach(entity => + { + String[] words = entity.Value.Split(); + foreach (string word in words) + { + entityWordBag.Add(word); + } + }); + + sentence.ForEach(token => { + if (!entityWordBag.Contains(token.Text)) + { + trainingTuple.Add(new TrainingData(token.Text, "O", token.Offset)); + } + }); + + entities.ForEach(entity => trainingTuple.Add(new TrainingData(entity.Value, entity.EntityName, entity.Start))); + + trainingTuple.Sort((left, right) => { + if (left.Start > right.Start) + return 1; + else if (left.Start < right.Start) + return -1; + else + return 0; + }); + + + + + return trainingTuple; + } + */ + + + + /* Train a model. + + * Parameters + * ---------- + * X : list of lists of dicts + Feature dicts for several documents (in a python-crfsuite format). + + * y : list of lists of strings + Labels for several documents. + + * X_dev : (optional) list of lists of dicts + Feature dicts used for testing. + + * y_dev : (optional) list of lists of strings + Labels corresponding to X_dev. + */ + public void Fit(List X, List y, List X_dev = null, List y_dev = null) { + Trainer trainer = new Trainer(); + for (int i = 0; i < Math.Min(X.Count, y.Count); i++) + { + // group ? + trainer.append(X[i], y[i], 0); + } + + trainer.train("model_test", X_dev == null ? -1 : 1); + } + + public Feature Word2Features(List sent, int i) { + string word = sent[i].Token; + string postag = sent[i].Tag; + + float bias = 1.0F; + String wordLower = word.ToLower(); + String wordLast3Char = wordLower.Length >= 3 ? wordLower.Substring(wordLower.Length - 3) : wordLower; + string patternAllCaptain = @"^[A-Z]+$"; + Boolean isSupper = new Regex(patternAllCaptain).IsMatch(word); + string patternFirstCaptain = @"^[A-Z]{1}[a-z]+$"; + Boolean isTitle = new Regex(patternFirstCaptain).IsMatch(word); + string patternAllDigit = @"^[0-9]+$"; + Boolean isDigit = new Regex(patternAllDigit).IsMatch(word); + String posTag = postag; + String postagFirst2Char = postag.Length >= 2 ? postag.Substring(0,2) : postag.Substring(0); + + Feature feature = new Feature(bias, wordLower, wordLast3Char, isSupper, isTitle, isDigit, posTag, postagFirst2Char); + + if (i > 0) + { + string minusWord = sent[i - 1].Token; + string minusPostag = sent[i - 1].Tag; + + feature.MinusWordLower = minusWord; + feature.MinusIsTitle = new Regex(patternFirstCaptain).IsMatch(minusWord); + feature.MinusIsSupper = new Regex(patternAllCaptain).IsMatch(minusWord); + feature.MinusPostag = minusPostag; + feature.MinusPostagFirst2Char = minusPostag.Length >= 2 ? minusPostag.Substring(0, 2) : minusPostag.Substring(0); + } + else { + feature.BOS = true; + } + + if ( i < sent.Count - 1) + { + string plusWord = sent[i + 1].Token; + string plusPostag = sent[i + 1].Tag; + + feature.PlusWordLower = plusWord; + feature.PlusIsTitle = new Regex(patternFirstCaptain).IsMatch(plusWord); + feature.PlusIsSupper = new Regex(patternAllCaptain).IsMatch(plusWord); + feature.PlusPostag = plusPostag; + feature.PlusPostagFirst2Char = plusPostag.Length >= 2 ? plusPostag.Substring(0, 2) : plusPostag.Substring(0); + } + return feature; + } + + public List sent2features(List sent) + { + List list = new List(); + for (int i = 0 ; i < sent.Count; i++ ) + { + list.Add(Word2Features(sent, i)); + } + return list; + } + + public List sent2labels(List sent) + { + List list = new List(); + sent.ForEach(tuple => list.Add(tuple.Entity)); + return list; + } + + public List sent2tokens(List sent) + { + List list = new List(); + sent.ForEach(tuple => list.Add(tuple.Token)); + return list; + } + } + public class Feature + { + public float Bias { get; set; } + public String WordLower { get; set; } + public String WordLast3Char { get; set; } + public Boolean IsSupper { get; set; } + public Boolean IsTitle { get; set; } + public Boolean IsDigit { get; set; } + public String Postag { get; set; } + public String PostagFirst2Char { get; set; } + + public Boolean BOS { get; set; } + public Boolean EOS { get; set; } + + public String PlusWordLower { get; set; } + public String PlusLast3Char { get; set; } + public Boolean PlusIsSupper { get; set; } + public Boolean PlusIsTitle { get; set; } + public Boolean PlusIsDigit { get; set; } + public String PlusPostag { get; set; } + public String PlusPostagFirst2Char { get; set; } + + public String MinusWordLower { get; set; } + public String MinusLast3Char { get; set; } + public Boolean MinusIsSupper { get; set; } + public Boolean MinusIsTitle { get; set; } + public Boolean MinusIsDigit { get; set; } + public String MinusPostag { get; set; } + public String MinusPostagFirst2Char { get; set; } + + public Feature(float bias, String wordLower, String wordLast3Char, Boolean isSupper, Boolean isTitle, Boolean isDigit, String posTag, String postagFirst2Char) + { + this.Bias = bias; + this.WordLower = wordLower; + this.WordLast3Char = wordLast3Char; + this.IsSupper = isSupper; + this.IsTitle = isTitle; + this.IsDigit = isDigit; + this.Postag = posTag; + this.PostagFirst2Char = postagFirst2Char; + } + } + + public class TrainingData + { + public String Token { get; set; } + public String Entity { get; set; } + public String Tag { get; set; } + + public TrainingData(string token, string entity, string tag) + { + this.Token = token; + this.Entity = entity; + this.Tag = tag; + } + } + + + public class Token + { + public String Text { get; set; } + public int Offset { get; set; } + public int End { get; set; } + } + + public class Entity + { + public String EntityName { get; set; } + public String Value { get; set; } + public int Start { get; set; } + public int End { get; set; } + } + + +} diff --git a/BotSharp.Core/Engines/CRFsuite/Item.cs b/BotSharp.Core/Engines/CRFsuite/Item.cs new file mode 100644 index 00000000..7844abb0 --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/Item.cs @@ -0,0 +1,312 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class Item : global::System.IDisposable, global::System.Collections.IEnumerable + , global::System.Collections.Generic.IEnumerable + { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal Item(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(Item obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~Item() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_Item(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public Item(global::System.Collections.ICollection c) : this() { + if (c == null) + throw new global::System.ArgumentNullException("c"); + foreach (Attribute element in c) { + this.Add(element); + } + } + + public bool IsFixedSize { + get { + return false; + } + } + + public bool IsReadOnly { + get { + return false; + } + } + + public Attribute this[int index] { + get { + return getitem(index); + } + set { + setitem(index, value); + } + } + + public int Capacity { + get { + return (int)capacity(); + } + set { + if (value < size()) + throw new global::System.ArgumentOutOfRangeException("Capacity"); + reserve((uint)value); + } + } + + public int Count { + get { + return (int)size(); + } + } + + public bool IsSynchronized { + get { + return false; + } + } + + public void CopyTo(Attribute[] array) + { + CopyTo(0, array, 0, this.Count); + } + + public void CopyTo(Attribute[] array, int arrayIndex) + { + CopyTo(0, array, arrayIndex, this.Count); + } + + public void CopyTo(int index, Attribute[] array, int arrayIndex, int count) + { + if (array == null) + throw new global::System.ArgumentNullException("array"); + if (index < 0) + throw new global::System.ArgumentOutOfRangeException("index", "Value is less than zero"); + if (arrayIndex < 0) + throw new global::System.ArgumentOutOfRangeException("arrayIndex", "Value is less than zero"); + if (count < 0) + throw new global::System.ArgumentOutOfRangeException("count", "Value is less than zero"); + if (array.Rank > 1) + throw new global::System.ArgumentException("Multi dimensional array.", "array"); + if (index+count > this.Count || arrayIndex+count > array.Length) + throw new global::System.ArgumentException("Number of elements to copy is too large."); + for (int i=0; i global::System.Collections.Generic.IEnumerable.GetEnumerator() { + return new ItemEnumerator(this); + } + + global::System.Collections.IEnumerator global::System.Collections.IEnumerable.GetEnumerator() { + return new ItemEnumerator(this); + } + + public ItemEnumerator GetEnumerator() { + return new ItemEnumerator(this); + } + + // Type-safe enumerator + /// Note that the IEnumerator documentation requires an InvalidOperationException to be thrown + /// whenever the collection is modified. This has been done for changes in the size of the + /// collection but not when one of the elements of the collection is modified as it is a bit + /// tricky to detect unmanaged code that modifies the collection under our feet. + public sealed class ItemEnumerator : global::System.Collections.IEnumerator + , global::System.Collections.Generic.IEnumerator + { + private Item collectionRef; + private int currentIndex; + private object currentObject; + private int currentSize; + + public ItemEnumerator(Item collection) { + collectionRef = collection; + currentIndex = -1; + currentObject = null; + currentSize = collectionRef.Count; + } + + // Type-safe iterator Current + public Attribute Current { + get { + if (currentIndex == -1) + throw new global::System.InvalidOperationException("Enumeration not started."); + if (currentIndex > currentSize - 1) + throw new global::System.InvalidOperationException("Enumeration finished."); + if (currentObject == null) + throw new global::System.InvalidOperationException("Collection modified."); + return (Attribute)currentObject; + } + } + + // Type-unsafe IEnumerator.Current + object global::System.Collections.IEnumerator.Current { + get { + return Current; + } + } + + public bool MoveNext() { + int size = collectionRef.Count; + bool moveOkay = (currentIndex+1 < size) && (size == currentSize); + if (moveOkay) { + currentIndex++; + currentObject = collectionRef[currentIndex]; + } else { + currentObject = null; + } + return moveOkay; + } + + public void Reset() { + currentIndex = -1; + currentObject = null; + if (collectionRef.Count != currentSize) { + throw new global::System.InvalidOperationException("Collection modified."); + } + } + + public void Dispose() { + currentIndex = -1; + currentObject = null; + } + } + + public void Clear() { + crfsuitePINVOKE.Item_Clear(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Add(Attribute x) { + crfsuitePINVOKE.Item_Add(swigCPtr, Attribute.getCPtr(x)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private uint size() { + uint ret = crfsuitePINVOKE.Item_size(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private uint capacity() { + uint ret = crfsuitePINVOKE.Item_capacity(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void reserve(uint n) { + crfsuitePINVOKE.Item_reserve(swigCPtr, n); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Item() : this(crfsuitePINVOKE.new_Item__SWIG_0(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Item(Item other) : this(crfsuitePINVOKE.new_Item__SWIG_1(Item.getCPtr(other)), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Item(int capacity) : this(crfsuitePINVOKE.new_Item__SWIG_2(capacity), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private Attribute getitemcopy(int index) { + Attribute ret = new Attribute(crfsuitePINVOKE.Item_getitemcopy(swigCPtr, index), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private Attribute getitem(int index) { + Attribute ret = new Attribute(crfsuitePINVOKE.Item_getitem(swigCPtr, index), false); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void setitem(int index, Attribute val) { + crfsuitePINVOKE.Item_setitem(swigCPtr, index, Attribute.getCPtr(val)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void AddRange(Item values) { + crfsuitePINVOKE.Item_AddRange(swigCPtr, Item.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public Item GetRange(int index, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.Item_GetRange(swigCPtr, index, count); + Item ret = (cPtr == global::System.IntPtr.Zero) ? null : new Item(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Insert(int index, Attribute x) { + crfsuitePINVOKE.Item_Insert(swigCPtr, index, Attribute.getCPtr(x)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void InsertRange(int index, Item values) { + crfsuitePINVOKE.Item_InsertRange(swigCPtr, index, Item.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveAt(int index) { + crfsuitePINVOKE.Item_RemoveAt(swigCPtr, index); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveRange(int index, int count) { + crfsuitePINVOKE.Item_RemoveRange(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public static Item Repeat(Attribute value, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.Item_Repeat(Attribute.getCPtr(value), count); + Item ret = (cPtr == global::System.IntPtr.Zero) ? null : new Item(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Reverse() { + crfsuitePINVOKE.Item_Reverse__SWIG_0(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Reverse(int index, int count) { + crfsuitePINVOKE.Item_Reverse__SWIG_1(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void SetRange(int index, Item values) { + crfsuitePINVOKE.Item_SetRange(swigCPtr, index, Item.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/ItemSequence.cs b/BotSharp.Core/Engines/CRFsuite/ItemSequence.cs new file mode 100644 index 00000000..12e7a9ae --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/ItemSequence.cs @@ -0,0 +1,312 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class ItemSequence : global::System.IDisposable, global::System.Collections.IEnumerable + , global::System.Collections.Generic.IEnumerable + { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal ItemSequence(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(ItemSequence obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~ItemSequence() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_ItemSequence(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public ItemSequence(global::System.Collections.ICollection c) : this() { + if (c == null) + throw new global::System.ArgumentNullException("c"); + foreach (Item element in c) { + this.Add(element); + } + } + + public bool IsFixedSize { + get { + return false; + } + } + + public bool IsReadOnly { + get { + return false; + } + } + + public Item this[int index] { + get { + return getitem(index); + } + set { + setitem(index, value); + } + } + + public int Capacity { + get { + return (int)capacity(); + } + set { + if (value < size()) + throw new global::System.ArgumentOutOfRangeException("Capacity"); + reserve((uint)value); + } + } + + public int Count { + get { + return (int)size(); + } + } + + public bool IsSynchronized { + get { + return false; + } + } + + public void CopyTo(Item[] array) + { + CopyTo(0, array, 0, this.Count); + } + + public void CopyTo(Item[] array, int arrayIndex) + { + CopyTo(0, array, arrayIndex, this.Count); + } + + public void CopyTo(int index, Item[] array, int arrayIndex, int count) + { + if (array == null) + throw new global::System.ArgumentNullException("array"); + if (index < 0) + throw new global::System.ArgumentOutOfRangeException("index", "Value is less than zero"); + if (arrayIndex < 0) + throw new global::System.ArgumentOutOfRangeException("arrayIndex", "Value is less than zero"); + if (count < 0) + throw new global::System.ArgumentOutOfRangeException("count", "Value is less than zero"); + if (array.Rank > 1) + throw new global::System.ArgumentException("Multi dimensional array.", "array"); + if (index+count > this.Count || arrayIndex+count > array.Length) + throw new global::System.ArgumentException("Number of elements to copy is too large."); + for (int i=0; i global::System.Collections.Generic.IEnumerable.GetEnumerator() { + return new ItemSequenceEnumerator(this); + } + + global::System.Collections.IEnumerator global::System.Collections.IEnumerable.GetEnumerator() { + return new ItemSequenceEnumerator(this); + } + + public ItemSequenceEnumerator GetEnumerator() { + return new ItemSequenceEnumerator(this); + } + + // Type-safe enumerator + /// Note that the IEnumerator documentation requires an InvalidOperationException to be thrown + /// whenever the collection is modified. This has been done for changes in the size of the + /// collection but not when one of the elements of the collection is modified as it is a bit + /// tricky to detect unmanaged code that modifies the collection under our feet. + public sealed class ItemSequenceEnumerator : global::System.Collections.IEnumerator + , global::System.Collections.Generic.IEnumerator + { + private ItemSequence collectionRef; + private int currentIndex; + private object currentObject; + private int currentSize; + + public ItemSequenceEnumerator(ItemSequence collection) { + collectionRef = collection; + currentIndex = -1; + currentObject = null; + currentSize = collectionRef.Count; + } + + // Type-safe iterator Current + public Item Current { + get { + if (currentIndex == -1) + throw new global::System.InvalidOperationException("Enumeration not started."); + if (currentIndex > currentSize - 1) + throw new global::System.InvalidOperationException("Enumeration finished."); + if (currentObject == null) + throw new global::System.InvalidOperationException("Collection modified."); + return (Item)currentObject; + } + } + + // Type-unsafe IEnumerator.Current + object global::System.Collections.IEnumerator.Current { + get { + return Current; + } + } + + public bool MoveNext() { + int size = collectionRef.Count; + bool moveOkay = (currentIndex+1 < size) && (size == currentSize); + if (moveOkay) { + currentIndex++; + currentObject = collectionRef[currentIndex]; + } else { + currentObject = null; + } + return moveOkay; + } + + public void Reset() { + currentIndex = -1; + currentObject = null; + if (collectionRef.Count != currentSize) { + throw new global::System.InvalidOperationException("Collection modified."); + } + } + + public void Dispose() { + currentIndex = -1; + currentObject = null; + } + } + + public void Clear() { + crfsuitePINVOKE.ItemSequence_Clear(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Add(Item x) { + crfsuitePINVOKE.ItemSequence_Add(swigCPtr, Item.getCPtr(x)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private uint size() { + uint ret = crfsuitePINVOKE.ItemSequence_size(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private uint capacity() { + uint ret = crfsuitePINVOKE.ItemSequence_capacity(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void reserve(uint n) { + crfsuitePINVOKE.ItemSequence_reserve(swigCPtr, n); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public ItemSequence() : this(crfsuitePINVOKE.new_ItemSequence__SWIG_0(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public ItemSequence(ItemSequence other) : this(crfsuitePINVOKE.new_ItemSequence__SWIG_1(ItemSequence.getCPtr(other)), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public ItemSequence(int capacity) : this(crfsuitePINVOKE.new_ItemSequence__SWIG_2(capacity), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private Item getitemcopy(int index) { + Item ret = new Item(crfsuitePINVOKE.ItemSequence_getitemcopy(swigCPtr, index), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private Item getitem(int index) { + Item ret = new Item(crfsuitePINVOKE.ItemSequence_getitem(swigCPtr, index), false); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void setitem(int index, Item val) { + crfsuitePINVOKE.ItemSequence_setitem(swigCPtr, index, Item.getCPtr(val)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void AddRange(ItemSequence values) { + crfsuitePINVOKE.ItemSequence_AddRange(swigCPtr, ItemSequence.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public ItemSequence GetRange(int index, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.ItemSequence_GetRange(swigCPtr, index, count); + ItemSequence ret = (cPtr == global::System.IntPtr.Zero) ? null : new ItemSequence(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Insert(int index, Item x) { + crfsuitePINVOKE.ItemSequence_Insert(swigCPtr, index, Item.getCPtr(x)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void InsertRange(int index, ItemSequence values) { + crfsuitePINVOKE.ItemSequence_InsertRange(swigCPtr, index, ItemSequence.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveAt(int index) { + crfsuitePINVOKE.ItemSequence_RemoveAt(swigCPtr, index); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveRange(int index, int count) { + crfsuitePINVOKE.ItemSequence_RemoveRange(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public static ItemSequence Repeat(Item value, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.ItemSequence_Repeat(Item.getCPtr(value), count); + ItemSequence ret = (cPtr == global::System.IntPtr.Zero) ? null : new ItemSequence(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Reverse() { + crfsuitePINVOKE.ItemSequence_Reverse__SWIG_0(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Reverse(int index, int count) { + crfsuitePINVOKE.ItemSequence_Reverse__SWIG_1(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void SetRange(int index, ItemSequence values) { + crfsuitePINVOKE.ItemSequence_SetRange(swigCPtr, index, ItemSequence.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/SWIGTYPE_p_void.cs b/BotSharp.Core/Engines/CRFsuite/SWIGTYPE_p_void.cs new file mode 100644 index 00000000..9ec7e20f --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/SWIGTYPE_p_void.cs @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class SWIGTYPE_p_void { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + + internal SWIGTYPE_p_void(global::System.IntPtr cPtr, bool futureUse) { + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + protected SWIGTYPE_p_void() { + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(SWIGTYPE_p_void obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } +} diff --git a/BotSharp.Core/Engines/CRFsuite/StringList.cs b/BotSharp.Core/Engines/CRFsuite/StringList.cs new file mode 100644 index 00000000..65c87e3b --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/StringList.cs @@ -0,0 +1,336 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class StringList : global::System.IDisposable, global::System.Collections.IEnumerable + , global::System.Collections.Generic.IList + { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal StringList(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(StringList obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~StringList() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_StringList(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public StringList(global::System.Collections.ICollection c) : this() { + if (c == null) + throw new global::System.ArgumentNullException("c"); + foreach (string element in c) { + this.Add(element); + } + } + + public bool IsFixedSize { + get { + return false; + } + } + + public bool IsReadOnly { + get { + return false; + } + } + + public string this[int index] { + get { + return getitem(index); + } + set { + setitem(index, value); + } + } + + public int Capacity { + get { + return (int)capacity(); + } + set { + if (value < size()) + throw new global::System.ArgumentOutOfRangeException("Capacity"); + reserve((uint)value); + } + } + + public int Count { + get { + return (int)size(); + } + } + + public bool IsSynchronized { + get { + return false; + } + } + + public void CopyTo(string[] array) + { + CopyTo(0, array, 0, this.Count); + } + + public void CopyTo(string[] array, int arrayIndex) + { + CopyTo(0, array, arrayIndex, this.Count); + } + + public void CopyTo(int index, string[] array, int arrayIndex, int count) + { + if (array == null) + throw new global::System.ArgumentNullException("array"); + if (index < 0) + throw new global::System.ArgumentOutOfRangeException("index", "Value is less than zero"); + if (arrayIndex < 0) + throw new global::System.ArgumentOutOfRangeException("arrayIndex", "Value is less than zero"); + if (count < 0) + throw new global::System.ArgumentOutOfRangeException("count", "Value is less than zero"); + if (array.Rank > 1) + throw new global::System.ArgumentException("Multi dimensional array.", "array"); + if (index+count > this.Count || arrayIndex+count > array.Length) + throw new global::System.ArgumentException("Number of elements to copy is too large."); + for (int i=0; i global::System.Collections.Generic.IEnumerable.GetEnumerator() { + return new StringListEnumerator(this); + } + + global::System.Collections.IEnumerator global::System.Collections.IEnumerable.GetEnumerator() { + return new StringListEnumerator(this); + } + + public StringListEnumerator GetEnumerator() { + return new StringListEnumerator(this); + } + + // Type-safe enumerator + /// Note that the IEnumerator documentation requires an InvalidOperationException to be thrown + /// whenever the collection is modified. This has been done for changes in the size of the + /// collection but not when one of the elements of the collection is modified as it is a bit + /// tricky to detect unmanaged code that modifies the collection under our feet. + public sealed class StringListEnumerator : global::System.Collections.IEnumerator + , global::System.Collections.Generic.IEnumerator + { + private StringList collectionRef; + private int currentIndex; + private object currentObject; + private int currentSize; + + public StringListEnumerator(StringList collection) { + collectionRef = collection; + currentIndex = -1; + currentObject = null; + currentSize = collectionRef.Count; + } + + // Type-safe iterator Current + public string Current { + get { + if (currentIndex == -1) + throw new global::System.InvalidOperationException("Enumeration not started."); + if (currentIndex > currentSize - 1) + throw new global::System.InvalidOperationException("Enumeration finished."); + if (currentObject == null) + throw new global::System.InvalidOperationException("Collection modified."); + return (string)currentObject; + } + } + + // Type-unsafe IEnumerator.Current + object global::System.Collections.IEnumerator.Current { + get { + return Current; + } + } + + public bool MoveNext() { + int size = collectionRef.Count; + bool moveOkay = (currentIndex+1 < size) && (size == currentSize); + if (moveOkay) { + currentIndex++; + currentObject = collectionRef[currentIndex]; + } else { + currentObject = null; + } + return moveOkay; + } + + public void Reset() { + currentIndex = -1; + currentObject = null; + if (collectionRef.Count != currentSize) { + throw new global::System.InvalidOperationException("Collection modified."); + } + } + + public void Dispose() { + currentIndex = -1; + currentObject = null; + } + } + + public void Clear() { + crfsuitePINVOKE.StringList_Clear(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Add(string x) { + crfsuitePINVOKE.StringList_Add(swigCPtr, x); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private uint size() { + uint ret = crfsuitePINVOKE.StringList_size(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private uint capacity() { + uint ret = crfsuitePINVOKE.StringList_capacity(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void reserve(uint n) { + crfsuitePINVOKE.StringList_reserve(swigCPtr, n); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList() : this(crfsuitePINVOKE.new_StringList__SWIG_0(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList(StringList other) : this(crfsuitePINVOKE.new_StringList__SWIG_1(StringList.getCPtr(other)), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList(int capacity) : this(crfsuitePINVOKE.new_StringList__SWIG_2(capacity), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private string getitemcopy(int index) { + string ret = crfsuitePINVOKE.StringList_getitemcopy(swigCPtr, index); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private string getitem(int index) { + string ret = crfsuitePINVOKE.StringList_getitem(swigCPtr, index); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + private void setitem(int index, string val) { + crfsuitePINVOKE.StringList_setitem(swigCPtr, index, val); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void AddRange(StringList values) { + crfsuitePINVOKE.StringList_AddRange(swigCPtr, StringList.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList GetRange(int index, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.StringList_GetRange(swigCPtr, index, count); + StringList ret = (cPtr == global::System.IntPtr.Zero) ? null : new StringList(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Insert(int index, string x) { + crfsuitePINVOKE.StringList_Insert(swigCPtr, index, x); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void InsertRange(int index, StringList values) { + crfsuitePINVOKE.StringList_InsertRange(swigCPtr, index, StringList.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveAt(int index) { + crfsuitePINVOKE.StringList_RemoveAt(swigCPtr, index); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void RemoveRange(int index, int count) { + crfsuitePINVOKE.StringList_RemoveRange(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public static StringList Repeat(string value, int count) { + global::System.IntPtr cPtr = crfsuitePINVOKE.StringList_Repeat(value, count); + StringList ret = (cPtr == global::System.IntPtr.Zero) ? null : new StringList(cPtr, true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void Reverse() { + crfsuitePINVOKE.StringList_Reverse__SWIG_0(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void Reverse(int index, int count) { + crfsuitePINVOKE.StringList_Reverse__SWIG_1(swigCPtr, index, count); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void SetRange(int index, StringList values) { + crfsuitePINVOKE.StringList_SetRange(swigCPtr, index, StringList.getCPtr(values)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public bool Contains(string value) { + bool ret = crfsuitePINVOKE.StringList_Contains(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public int IndexOf(string value) { + int ret = crfsuitePINVOKE.StringList_IndexOf(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public int LastIndexOf(string value) { + int ret = crfsuitePINVOKE.StringList_LastIndexOf(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public bool Remove(string value) { + bool ret = crfsuitePINVOKE.StringList_Remove(swigCPtr, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/Tagger.cs b/BotSharp.Core/Engines/CRFsuite/Tagger.cs new file mode 100644 index 00000000..f1f3cfdd --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/Tagger.cs @@ -0,0 +1,98 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class Tagger : global::System.IDisposable { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal Tagger(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(Tagger obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~Tagger() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_Tagger(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public Tagger() : this(crfsuitePINVOKE.new_Tagger(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public bool open(string name) { + bool ret = crfsuitePINVOKE.Tagger_open__SWIG_0(swigCPtr, name); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public bool open(SWIGTYPE_p_void data, uint size) { + bool ret = crfsuitePINVOKE.Tagger_open__SWIG_1(swigCPtr, SWIGTYPE_p_void.getCPtr(data), size); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void close() { + crfsuitePINVOKE.Tagger_close(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList labels() { + StringList ret = new StringList(crfsuitePINVOKE.Tagger_labels(swigCPtr), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public StringList tag(ItemSequence xseq) { + StringList ret = new StringList(crfsuitePINVOKE.Tagger_tag(swigCPtr, ItemSequence.getCPtr(xseq)), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void set(ItemSequence xseq) { + crfsuitePINVOKE.Tagger_set(swigCPtr, ItemSequence.getCPtr(xseq)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public StringList viterbi() { + StringList ret = new StringList(crfsuitePINVOKE.Tagger_viterbi(swigCPtr), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public double probability(StringList yseq) { + double ret = crfsuitePINVOKE.Tagger_probability(swigCPtr, StringList.getCPtr(yseq)); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public double marginal(string y, int t) { + double ret = crfsuitePINVOKE.Tagger_marginal(swigCPtr, y, t); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/Trainer.cs b/BotSharp.Core/Engines/CRFsuite/Trainer.cs new file mode 100644 index 00000000..f4871783 --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/Trainer.cs @@ -0,0 +1,118 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class Trainer : global::System.IDisposable { + private global::System.Runtime.InteropServices.HandleRef swigCPtr; + protected bool swigCMemOwn; + + internal Trainer(global::System.IntPtr cPtr, bool cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(this, cPtr); + } + + internal static global::System.Runtime.InteropServices.HandleRef getCPtr(Trainer obj) { + return (obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr; + } + + ~Trainer() { + Dispose(); + } + + public virtual void Dispose() { + lock(this) { + if (swigCPtr.Handle != global::System.IntPtr.Zero) { + if (swigCMemOwn) { + swigCMemOwn = false; + crfsuitePINVOKE.delete_Trainer(swigCPtr); + } + swigCPtr = new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero); + } + global::System.GC.SuppressFinalize(this); + } + } + + public Trainer() : this(crfsuitePINVOKE.new_Trainer(), true) { + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + SwigDirectorConnect(); + } + + public void clear() { + crfsuitePINVOKE.Trainer_clear(swigCPtr); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public void append(ItemSequence xseq, StringList yseq, int group) { + crfsuitePINVOKE.Trainer_append(swigCPtr, ItemSequence.getCPtr(xseq), StringList.getCPtr(yseq), group); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public bool select(string algorithm, string type) { + bool ret = crfsuitePINVOKE.Trainer_select(swigCPtr, algorithm, type); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public int train(string model, int holdout) { + int ret = crfsuitePINVOKE.Trainer_train(swigCPtr, model, holdout); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public StringList params_() { + StringList ret = new StringList(crfsuitePINVOKE.Trainer_params_(swigCPtr), true); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public void set(string name, string value) { + crfsuitePINVOKE.Trainer_set(swigCPtr, name, value); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + public string get(string name) { + string ret = crfsuitePINVOKE.Trainer_get(swigCPtr, name); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public string help(string name) { + string ret = crfsuitePINVOKE.Trainer_help(swigCPtr, name); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + + public virtual void message(string msg) { + if (SwigDerivedClassHasMethod("message", swigMethodTypes0)) crfsuitePINVOKE.Trainer_messageSwigExplicitTrainer(swigCPtr, msg); else crfsuitePINVOKE.Trainer_message(swigCPtr, msg); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + } + + private void SwigDirectorConnect() { + if (SwigDerivedClassHasMethod("message", swigMethodTypes0)) + swigDelegate0 = new SwigDelegateTrainer_0(SwigDirectormessage); + crfsuitePINVOKE.Trainer_director_connect(swigCPtr, swigDelegate0); + } + + private bool SwigDerivedClassHasMethod(string methodName, global::System.Type[] methodTypes) { + global::System.Reflection.MethodInfo methodInfo = this.GetType().GetMethod(methodName, global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.NonPublic | global::System.Reflection.BindingFlags.Instance, null, methodTypes, null); + bool hasDerivedMethod = methodInfo.DeclaringType.IsSubclassOf(typeof(Trainer)); + return hasDerivedMethod; + } + + private void SwigDirectormessage(string msg) { + message(msg); + } + + public delegate void SwigDelegateTrainer_0(string msg); + + private SwigDelegateTrainer_0 swigDelegate0; + + private static global::System.Type[] swigMethodTypes0 = new global::System.Type[] { typeof(string) }; +} diff --git a/BotSharp.Core/Engines/CRFsuite/crfsuite.cs b/BotSharp.Core/Engines/CRFsuite/crfsuite.cs new file mode 100644 index 00000000..0f0b4f13 --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/crfsuite.cs @@ -0,0 +1,19 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +public class crfsuite { + public static string version() { + string ret = crfsuitePINVOKE.version(); + if (crfsuitePINVOKE.SWIGPendingException.Pending) throw crfsuitePINVOKE.SWIGPendingException.Retrieve(); + return ret; + } + +} diff --git a/BotSharp.Core/Engines/CRFsuite/crfsuitePINVOKE.cs b/BotSharp.Core/Engines/CRFsuite/crfsuitePINVOKE.cs new file mode 100644 index 00000000..75eee0a8 --- /dev/null +++ b/BotSharp.Core/Engines/CRFsuite/crfsuitePINVOKE.cs @@ -0,0 +1,498 @@ +//------------------------------------------------------------------------------ +// +// +// This file was automatically generated by SWIG (http://www.swig.org). +// Version 3.0.12 +// +// Do not make changes to this file unless you know what you are doing--modify +// the SWIG interface file instead. +//------------------------------------------------------------------------------ + + +class crfsuitePINVOKE { + + protected class SWIGExceptionHelper { + + public delegate void ExceptionDelegate(string message); + public delegate void ExceptionArgumentDelegate(string message, string paramName); + + static ExceptionDelegate applicationDelegate = new ExceptionDelegate(SetPendingApplicationException); + static ExceptionDelegate arithmeticDelegate = new ExceptionDelegate(SetPendingArithmeticException); + static ExceptionDelegate divideByZeroDelegate = new ExceptionDelegate(SetPendingDivideByZeroException); + static ExceptionDelegate indexOutOfRangeDelegate = new ExceptionDelegate(SetPendingIndexOutOfRangeException); + static ExceptionDelegate invalidCastDelegate = new ExceptionDelegate(SetPendingInvalidCastException); + static ExceptionDelegate invalidOperationDelegate = new ExceptionDelegate(SetPendingInvalidOperationException); + static ExceptionDelegate ioDelegate = new ExceptionDelegate(SetPendingIOException); + static ExceptionDelegate nullReferenceDelegate = new ExceptionDelegate(SetPendingNullReferenceException); + static ExceptionDelegate outOfMemoryDelegate = new ExceptionDelegate(SetPendingOutOfMemoryException); + static ExceptionDelegate overflowDelegate = new ExceptionDelegate(SetPendingOverflowException); + static ExceptionDelegate systemDelegate = new ExceptionDelegate(SetPendingSystemException); + + static ExceptionArgumentDelegate argumentDelegate = new ExceptionArgumentDelegate(SetPendingArgumentException); + static ExceptionArgumentDelegate argumentNullDelegate = new ExceptionArgumentDelegate(SetPendingArgumentNullException); + static ExceptionArgumentDelegate argumentOutOfRangeDelegate = new ExceptionArgumentDelegate(SetPendingArgumentOutOfRangeException); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="SWIGRegisterExceptionCallbacks_crfsuite")] + public static extern void SWIGRegisterExceptionCallbacks_crfsuite( + ExceptionDelegate applicationDelegate, + ExceptionDelegate arithmeticDelegate, + ExceptionDelegate divideByZeroDelegate, + ExceptionDelegate indexOutOfRangeDelegate, + ExceptionDelegate invalidCastDelegate, + ExceptionDelegate invalidOperationDelegate, + ExceptionDelegate ioDelegate, + ExceptionDelegate nullReferenceDelegate, + ExceptionDelegate outOfMemoryDelegate, + ExceptionDelegate overflowDelegate, + ExceptionDelegate systemExceptionDelegate); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="SWIGRegisterExceptionArgumentCallbacks_crfsuite")] + public static extern void SWIGRegisterExceptionCallbacksArgument_crfsuite( + ExceptionArgumentDelegate argumentDelegate, + ExceptionArgumentDelegate argumentNullDelegate, + ExceptionArgumentDelegate argumentOutOfRangeDelegate); + + static void SetPendingApplicationException(string message) { + SWIGPendingException.Set(new global::System.ApplicationException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingArithmeticException(string message) { + SWIGPendingException.Set(new global::System.ArithmeticException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingDivideByZeroException(string message) { + SWIGPendingException.Set(new global::System.DivideByZeroException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingIndexOutOfRangeException(string message) { + SWIGPendingException.Set(new global::System.IndexOutOfRangeException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingInvalidCastException(string message) { + SWIGPendingException.Set(new global::System.InvalidCastException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingInvalidOperationException(string message) { + SWIGPendingException.Set(new global::System.InvalidOperationException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingIOException(string message) { + SWIGPendingException.Set(new global::System.IO.IOException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingNullReferenceException(string message) { + SWIGPendingException.Set(new global::System.NullReferenceException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingOutOfMemoryException(string message) { + SWIGPendingException.Set(new global::System.OutOfMemoryException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingOverflowException(string message) { + SWIGPendingException.Set(new global::System.OverflowException(message, SWIGPendingException.Retrieve())); + } + static void SetPendingSystemException(string message) { + SWIGPendingException.Set(new global::System.SystemException(message, SWIGPendingException.Retrieve())); + } + + static void SetPendingArgumentException(string message, string paramName) { + SWIGPendingException.Set(new global::System.ArgumentException(message, paramName, SWIGPendingException.Retrieve())); + } + static void SetPendingArgumentNullException(string message, string paramName) { + global::System.Exception e = SWIGPendingException.Retrieve(); + if (e != null) message = message + " Inner Exception: " + e.Message; + SWIGPendingException.Set(new global::System.ArgumentNullException(paramName, message)); + } + static void SetPendingArgumentOutOfRangeException(string message, string paramName) { + global::System.Exception e = SWIGPendingException.Retrieve(); + if (e != null) message = message + " Inner Exception: " + e.Message; + SWIGPendingException.Set(new global::System.ArgumentOutOfRangeException(paramName, message)); + } + + static SWIGExceptionHelper() { + SWIGRegisterExceptionCallbacks_crfsuite( + applicationDelegate, + arithmeticDelegate, + divideByZeroDelegate, + indexOutOfRangeDelegate, + invalidCastDelegate, + invalidOperationDelegate, + ioDelegate, + nullReferenceDelegate, + outOfMemoryDelegate, + overflowDelegate, + systemDelegate); + + SWIGRegisterExceptionCallbacksArgument_crfsuite( + argumentDelegate, + argumentNullDelegate, + argumentOutOfRangeDelegate); + } + } + + protected static SWIGExceptionHelper swigExceptionHelper = new SWIGExceptionHelper(); + + public class SWIGPendingException { + [global::System.ThreadStatic] + private static global::System.Exception pendingException = null; + private static int numExceptionsPending = 0; + + public static bool Pending { + get { + bool pending = false; + if (numExceptionsPending > 0) + if (pendingException != null) + pending = true; + return pending; + } + } + + public static void Set(global::System.Exception e) { + if (pendingException != null) + throw new global::System.ApplicationException("FATAL: An earlier pending exception from unmanaged code was missed and thus not thrown (" + pendingException.ToString() + ")", e); + pendingException = e; + lock(typeof(crfsuitePINVOKE)) { + numExceptionsPending++; + } + } + + public static global::System.Exception Retrieve() { + global::System.Exception e = null; + if (numExceptionsPending > 0) { + if (pendingException != null) { + e = pendingException; + pendingException = null; + lock(typeof(crfsuitePINVOKE)) { + numExceptionsPending--; + } + } + } + return e; + } + } + + + protected class SWIGStringHelper { + + public delegate string SWIGStringDelegate(string message); + static SWIGStringDelegate stringDelegate = new SWIGStringDelegate(CreateString); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="SWIGRegisterStringCallback_crfsuite")] + public static extern void SWIGRegisterStringCallback_crfsuite(SWIGStringDelegate stringDelegate); + + static string CreateString(string cString) { + return cString; + } + + static SWIGStringHelper() { + SWIGRegisterStringCallback_crfsuite(stringDelegate); + } + } + + static protected SWIGStringHelper swigStringHelper = new SWIGStringHelper(); + + + static crfsuitePINVOKE() { + } + + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Attribute_attr_set")] + public static extern void Attribute_attr_set(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Attribute_attr_get")] + public static extern string Attribute_attr_get(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Attribute_value_set")] + public static extern void Attribute_value_set(global::System.Runtime.InteropServices.HandleRef jarg1, double jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Attribute_value_get")] + public static extern double Attribute_value_get(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Attribute__SWIG_0")] + public static extern global::System.IntPtr new_Attribute__SWIG_0(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Attribute__SWIG_1")] + public static extern global::System.IntPtr new_Attribute__SWIG_1(string jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Attribute__SWIG_2")] + public static extern global::System.IntPtr new_Attribute__SWIG_2(string jarg1, double jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_Attribute")] + public static extern void delete_Attribute(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Trainer")] + public static extern global::System.IntPtr new_Trainer(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_Trainer")] + public static extern void delete_Trainer(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_clear")] + public static extern void Trainer_clear(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_append")] + public static extern void Trainer_append(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2, global::System.Runtime.InteropServices.HandleRef jarg3, int jarg4); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_select")] + public static extern bool Trainer_select(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2, string jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_train")] + public static extern int Trainer_train(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_params_")] + public static extern global::System.IntPtr Trainer_params_(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_set")] + public static extern void Trainer_set(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2, string jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_get")] + public static extern string Trainer_get(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_help")] + public static extern string Trainer_help(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_message")] + public static extern void Trainer_message(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_messageSwigExplicitTrainer")] + public static extern void Trainer_messageSwigExplicitTrainer(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Trainer_director_connect")] + public static extern void Trainer_director_connect(global::System.Runtime.InteropServices.HandleRef jarg1, Trainer.SwigDelegateTrainer_0 delegate0); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Tagger")] + public static extern global::System.IntPtr new_Tagger(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_Tagger")] + public static extern void delete_Tagger(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_open__SWIG_0")] + public static extern bool Tagger_open__SWIG_0(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_open__SWIG_1")] + public static extern bool Tagger_open__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2, uint jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_close")] + public static extern void Tagger_close(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_labels")] + public static extern global::System.IntPtr Tagger_labels(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_tag")] + public static extern global::System.IntPtr Tagger_tag(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_set")] + public static extern void Tagger_set(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_viterbi")] + public static extern global::System.IntPtr Tagger_viterbi(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_probability")] + public static extern double Tagger_probability(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Tagger_marginal")] + public static extern double Tagger_marginal(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_version")] + public static extern string version(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Clear")] + public static extern void Item_Clear(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Add")] + public static extern void Item_Add(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_size")] + public static extern uint Item_size(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_capacity")] + public static extern uint Item_capacity(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_reserve")] + public static extern void Item_reserve(global::System.Runtime.InteropServices.HandleRef jarg1, uint jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Item__SWIG_0")] + public static extern global::System.IntPtr new_Item__SWIG_0(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Item__SWIG_1")] + public static extern global::System.IntPtr new_Item__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_Item__SWIG_2")] + public static extern global::System.IntPtr new_Item__SWIG_2(int jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_getitemcopy")] + public static extern global::System.IntPtr Item_getitemcopy(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_getitem")] + public static extern global::System.IntPtr Item_getitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_setitem")] + public static extern void Item_setitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_AddRange")] + public static extern void Item_AddRange(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_GetRange")] + public static extern global::System.IntPtr Item_GetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Insert")] + public static extern void Item_Insert(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_InsertRange")] + public static extern void Item_InsertRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_RemoveAt")] + public static extern void Item_RemoveAt(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_RemoveRange")] + public static extern void Item_RemoveRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Repeat")] + public static extern global::System.IntPtr Item_Repeat(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Reverse__SWIG_0")] + public static extern void Item_Reverse__SWIG_0(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_Reverse__SWIG_1")] + public static extern void Item_Reverse__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_Item_SetRange")] + public static extern void Item_SetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_Item")] + public static extern void delete_Item(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Clear")] + public static extern void ItemSequence_Clear(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Add")] + public static extern void ItemSequence_Add(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_size")] + public static extern uint ItemSequence_size(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_capacity")] + public static extern uint ItemSequence_capacity(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_reserve")] + public static extern void ItemSequence_reserve(global::System.Runtime.InteropServices.HandleRef jarg1, uint jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_ItemSequence__SWIG_0")] + public static extern global::System.IntPtr new_ItemSequence__SWIG_0(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_ItemSequence__SWIG_1")] + public static extern global::System.IntPtr new_ItemSequence__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_ItemSequence__SWIG_2")] + public static extern global::System.IntPtr new_ItemSequence__SWIG_2(int jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_getitemcopy")] + public static extern global::System.IntPtr ItemSequence_getitemcopy(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_getitem")] + public static extern global::System.IntPtr ItemSequence_getitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_setitem")] + public static extern void ItemSequence_setitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_AddRange")] + public static extern void ItemSequence_AddRange(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_GetRange")] + public static extern global::System.IntPtr ItemSequence_GetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Insert")] + public static extern void ItemSequence_Insert(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_InsertRange")] + public static extern void ItemSequence_InsertRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_RemoveAt")] + public static extern void ItemSequence_RemoveAt(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_RemoveRange")] + public static extern void ItemSequence_RemoveRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Repeat")] + public static extern global::System.IntPtr ItemSequence_Repeat(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Reverse__SWIG_0")] + public static extern void ItemSequence_Reverse__SWIG_0(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_Reverse__SWIG_1")] + public static extern void ItemSequence_Reverse__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_ItemSequence_SetRange")] + public static extern void ItemSequence_SetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_ItemSequence")] + public static extern void delete_ItemSequence(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Clear")] + public static extern void StringList_Clear(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Add")] + public static extern void StringList_Add(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_size")] + public static extern uint StringList_size(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_capacity")] + public static extern uint StringList_capacity(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_reserve")] + public static extern void StringList_reserve(global::System.Runtime.InteropServices.HandleRef jarg1, uint jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_StringList__SWIG_0")] + public static extern global::System.IntPtr new_StringList__SWIG_0(); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_StringList__SWIG_1")] + public static extern global::System.IntPtr new_StringList__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_new_StringList__SWIG_2")] + public static extern global::System.IntPtr new_StringList__SWIG_2(int jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_getitemcopy")] + public static extern string StringList_getitemcopy(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_getitem")] + public static extern string StringList_getitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_setitem")] + public static extern void StringList_setitem(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, string jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_AddRange")] + public static extern void StringList_AddRange(global::System.Runtime.InteropServices.HandleRef jarg1, global::System.Runtime.InteropServices.HandleRef jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_GetRange")] + public static extern global::System.IntPtr StringList_GetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Insert")] + public static extern void StringList_Insert(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, string jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_InsertRange")] + public static extern void StringList_InsertRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_RemoveAt")] + public static extern void StringList_RemoveAt(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_RemoveRange")] + public static extern void StringList_RemoveRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Repeat")] + public static extern global::System.IntPtr StringList_Repeat(string jarg1, int jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Reverse__SWIG_0")] + public static extern void StringList_Reverse__SWIG_0(global::System.Runtime.InteropServices.HandleRef jarg1); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Reverse__SWIG_1")] + public static extern void StringList_Reverse__SWIG_1(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, int jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_SetRange")] + public static extern void StringList_SetRange(global::System.Runtime.InteropServices.HandleRef jarg1, int jarg2, global::System.Runtime.InteropServices.HandleRef jarg3); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Contains")] + public static extern bool StringList_Contains(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_IndexOf")] + public static extern int StringList_IndexOf(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_LastIndexOf")] + public static extern int StringList_LastIndexOf(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_StringList_Remove")] + public static extern bool StringList_Remove(global::System.Runtime.InteropServices.HandleRef jarg1, string jarg2); + + [global::System.Runtime.InteropServices.DllImport("crfsuite", EntryPoint="CSharp_delete_StringList")] + public static extern void delete_StringList(global::System.Runtime.InteropServices.HandleRef jarg1); +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs index d0fd5587..a1862712 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs @@ -1,11 +1,13 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; +using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace BotSharp.Core.Engines.SpaCy @@ -22,12 +24,29 @@ namespace BotSharp.Core.Engines.SpaCy String outputDir = "./entity_rec_output2"; int iterTimes = 20; - agent.Entities.ForEach(entity => entitiesInTrainingSet.Add(entity.Name)); + List trainingData = new List(); + + var dc = new DefaultDataContextLoader().GetDefaultDc(); + var corpus = agent.GrabCorpus(dc); + + corpus.UserSays.ForEach(userSay => + { + if (userSay.Entities != null) { + //texts.Add(userSay.Text); + List entityLabel = new List(); + userSay.Entities.ForEach(entity => { + entityLabel.Add(new EntityLabel(entity.Start, entity.End, entity.Entity)); + entitiesInTrainingSet.Add(entity.Entity); + }); + trainingData.Add(new TrainingNode(userSay.Text, entityLabel)); + } + }); + entitiesInTrainingSet = entitiesInTrainingSet.Distinct().ToList(); var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("entityrecognizer", Method.POST); request.RequestFormat = DataFormat.Json; - request.AddParameter("application/json", JsonConvert.SerializeObject(new { ModelPath = modelPath, NewModelName = newModelName, OutputDir = outputDir, IterTimes = iterTimes, EntitiesInTrainingSet = entitiesInTrainingSet }), ParameterType.RequestBody); + request.AddParameter("application/json", JsonConvert.SerializeObject(new NERTrainingModel( modelPath, newModelName, outputDir, iterTimes, trainingData, entitiesInTrainingSet)), ParameterType.RequestBody); var response = client.Execute(request); @@ -41,4 +60,57 @@ namespace BotSharp.Core.Engines.SpaCy { public Boolean EntityModelTrained { get; set; } } + + public class EntityLabel + { + public EntityLabel(int start, int end, string entity) + { + this.Start = start; + this.End = end; + this.Name = entity; + } + + public int Start { get; set; } + public int End { get; set; } + public String Name { get; set; } + + } + + public class TrainingNode + { + public TrainingNode(string text, List entityLabel) + { + this.Text = text; + this.Labels = entityLabel; + } + + public String Text { get; set; } + public List Labels { get; set; } + + } + + public class NERTrainingModel + { + public NERTrainingModel(string modelPath, string newModelName, string outputDir, int iterTimes, List trainingData, List entitiesInTrainingSet) + { + this.ModelPath = modelPath; + this.NewModelName = newModelName; + this.OutputDir = outputDir; + this.IterTimes = iterTimes; + this.TrainingData = trainingData; + this.EntitiesInTrainingSet = entitiesInTrainingSet; + } + + public string ModelPath { set; get; } + + public string NewModelName { set; get; } + + public string OutputDir { set; get; } + + public int IterTimes { set; get; } + + public List TrainingData { set; get; } + + public List EntitiesInTrainingSet { set;get;} + } } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs new file mode 100644 index 00000000..4c7c56f7 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs @@ -0,0 +1,20 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines.SpaCy +{ + class SpaCyTagger : INlpPipeline + { + public IConfiguration Configuration { get; set; } + + public bool Process(Agent agent, JObject data) + { + throw new NotImplementedException(); + } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs index bb8ccfde..dc95802a 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs @@ -53,12 +53,12 @@ namespace BotSharp.Core.Engines.SpaCy var response = client.Execute(request); data["ModelName"] = response.Data.ModelName; - + /* //Predict var request2 = new RestRequest("predict", Method.GET); request2.AddParameter("text", "the roof is leaking"); var response2 = client.Execute(request2); - + */ return true; } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 87d3ed15..b36e19d1 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -1,11 +1,13 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Core.Models; +using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace BotSharp.Core.Engines.SpaCy @@ -18,13 +20,26 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenize", Method.GET); - request.AddParameter("text", ""); - var response = client.Execute(request); + List> tokens = new List>(); + Boolean res = true; + var dc = new DefaultDataContextLoader().GetDefaultDc(); + var corpus = agent.GrabCorpus(dc); - data.Add("Tokens", JToken.FromObject(response.Data.Tokens)); + corpus.UserSays.ForEach(usersay => { + request.AddParameter("text", usersay.Text); + var response = client.Execute(request); + tokens.Add(response.Data.Tokens); + res = res && response.IsSuccessful; + }); - return response.IsSuccessful; + + + + data.Add("Tokens", JToken.FromObject(tokens)); + + return res; } + public class Result { diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs index e1cdc6c3..1602141f 100644 --- a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Text; using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; +using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using RestSharp; @@ -17,12 +18,21 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("featurize", Method.GET); - request.AddParameter("text", ""); - var response = client.Execute(request); + List> vectors = new List>(); + Boolean res = true; + var dc = new DefaultDataContextLoader().GetDefaultDc(); + var corpus = agent.GrabCorpus(dc); - data.Add("Features", JToken.FromObject(response.Data.Vectors)); + corpus.UserSays.ForEach(usersay => { + request.AddParameter("text", usersay.Text); + var response = client.Execute(request); + vectors.Add(response.Data.Vectors); + res = res && response.IsSuccessful; + }); - return response.IsSuccessful; + data.Add("Features", JToken.FromObject(vectors)); + + return res; } public class Result diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index 1cbbe994..f96713c9 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -9,6 +9,6 @@ "SpaCyProvider": { "Url": "http://10.2.21.200:5005" }, - "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer" + "Pipe": "SpaCyTokenizer, SpacyFeaturizer, CRFsuiteEntityRecognizer" //SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer } }