diff --git a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj index 78982f41..c4c7e758 100644 --- a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj +++ b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj @@ -8,4 +8,10 @@ + + + ..\..\..\..\..\..\Program Files\dotnet\sdk\NuGetFallbackFolder\microsoft.data.sqlite.core\2.1.0\lib\netstandard2.0\Microsoft.Data.Sqlite.dll + + + diff --git a/BotSharp.MachineLearning/Entropy/AbstractDataIndexer.cs b/BotSharp.MachineLearning/Entropy/AbstractDataIndexer.cs new file mode 100644 index 00000000..ac59d886 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/AbstractDataIndexer.cs @@ -0,0 +1,230 @@ +// Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the AbstractDataIndexer.java source file found in the +//original java implementation of MaxEnt. + +using System; +using System.Collections.Generic; + +namespace BotSharp.MachineLearning +{ + /// + /// Abstract base for DataIndexer implementations. + /// + /// + /// Tom Morton + /// + /// + /// Richard J. Northedge + /// + public abstract class AbstractDataIndexer : ITrainingDataIndexer + { + private int[][] mContexts; + private int[] mOutcomeList; + private int[] mNumTimesEventsSeen; + private string[] mPredicateLabels; + private string[] mOutcomeLabels; + + /// + /// Gets an array of context data calculated from the training data. + /// + /// + /// Array of integer arrays, each containing the context data for an event. + /// + public virtual int[][] GetContexts() + { + return mContexts; + } + + /// + /// Sets the array of context data calculated from the training data. + /// + /// + /// Array of integer arrays, each containing the context data for an event. + /// + protected internal void SetContexts(int[][] newContexts) + { + mContexts = newContexts; + } + + /// + /// Gets an array indicating how many times each event is seen. + /// + /// + /// Integer array with event frequencies. + /// + public virtual int[] GetNumTimesEventsSeen() + { + return mNumTimesEventsSeen; + } + + /// + /// Sets an array indicating how many times each event is seen. + /// + /// + /// Integer array with event frequencies. + /// + protected internal void SetNumTimesEventsSeen(int[] newNumTimesEventsSeen) + { + mNumTimesEventsSeen = newNumTimesEventsSeen; + } + + /// + /// Gets an outcome list. + /// + /// + /// Integer array of outcomes. + /// + public virtual int[] GetOutcomeList() + { + return mOutcomeList; + } + + /// + /// Sets an outcome list. + /// + /// + /// Integer array of outcomes. + /// + protected internal void SetOutcomeList(int[] newOutcomeList) + { + mOutcomeList = newOutcomeList; + } + + /// + /// Gets an array of predicate labels. + /// + /// + /// Array of predicate labels. + /// + public virtual string[] GetPredicateLabels() + { + return mPredicateLabels; + } + + /// + /// Sets an array of predicate labels. + /// + /// + /// Array of predicate labels. + /// + protected internal void SetPredicateLabels(string[] newPredicateLabels) + { + mPredicateLabels = newPredicateLabels; + } + + /// + /// Gets an array of outcome labels. + /// + /// + /// Array of outcome labels. + /// + public virtual string[] GetOutcomeLabels() + { + return mOutcomeLabels; + } + + /// + /// Sets an array of outcome labels. + /// + /// + /// Array of outcome labels. + /// + protected internal void SetOutcomeLabels(string[] newOutcomeLabels) + { + mOutcomeLabels = newOutcomeLabels; + } + + /// + /// Sorts and uniques the array of comparable events. This method + /// will alter the eventsToCompare array -- it does an in place + /// sort, followed by an in place edit to remove duplicates. + /// + /// + /// a List of ComparableEvent values + /// + protected internal virtual void SortAndMerge(List eventsToCompare) + { + eventsToCompare.Sort(); + int eventCount = eventsToCompare.Count; + int uniqueEventCount = 1; // assertion: eventsToCompare.length >= 1 + + if (eventCount <= 1) + { + return; // nothing to do; edge case (see assertion) + } + + ComparableEvent comparableEvent = eventsToCompare[0]; + for (int currentEvent = 1; currentEvent < eventCount; currentEvent++) + { + ComparableEvent eventToCompare = eventsToCompare[currentEvent]; + + if (comparableEvent.Equals(eventToCompare)) + { + comparableEvent.SeenCount++; // increment the seen count + eventsToCompare[currentEvent] = null; // kill the duplicate + } + else + { + comparableEvent = eventToCompare; // a new champion emerges... + uniqueEventCount++; // increment the # of unique events + } + } + + //NotifyProgress("done. Reduced " + eventCount + " events to " + uniqueEventCount + "."); + + mContexts = new int[uniqueEventCount][]; + mOutcomeList = new int[uniqueEventCount]; + mNumTimesEventsSeen = new int[uniqueEventCount]; + + for (int currentEvent = 0, currentStoredEvent = 0; currentEvent < eventCount; currentEvent++) + { + ComparableEvent eventToStore = eventsToCompare[currentEvent]; + if (null == eventToStore) + { + continue; // this was a dupe, skip over it. + } + mNumTimesEventsSeen[currentStoredEvent] = eventToStore.SeenCount; + mOutcomeList[currentStoredEvent] = eventToStore.Outcome; + mContexts[currentStoredEvent] = eventToStore.GetPredicateIndexes(); + ++currentStoredEvent; + } + } + + /// + /// Utility method for creating a string[] array from a dictionary whose + /// keys are labels (strings) to be stored in the array and whose + /// values are the indices (integers) at which the corresponding + /// labels should be inserted. + /// + /// + /// a Dictionary value + /// + /// + /// a string[] value + /// + protected internal static string[] ToIndexedStringArray(Dictionary labelToIndexMap) + { + string[] indexedArray = new string[labelToIndexMap.Count]; + int[] indices = new int[labelToIndexMap.Count]; + labelToIndexMap.Keys.CopyTo(indexedArray, 0); + labelToIndexMap.Values.CopyTo(indices, 0); + Array.Sort(indices, indexedArray); + return indexedArray; + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/BasicContextGenerator.cs b/BotSharp.MachineLearning/Entropy/BasicContextGenerator.cs new file mode 100644 index 00000000..813e3dd1 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/BasicContextGenerator.cs @@ -0,0 +1,70 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the BasicContextGenerator.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// Generate contexts for maxent decisions, assuming that the input + /// given to the GetContext() method is a string containing contextual + /// predicates separated by spaces, e.g: + ///

+ /// cp_1 cp_2 ... cp_n + ///

+ /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// based on BasicContextGenerator.java, $Revision: 1.2 $, $Date: 2002/04/30 08:48:35 $ + /// + public class BasicContextGenerator : IContextGenerator + { + /// + /// Builds up the list of contextual predicates given a string. + /// + /// + /// string with contextual predicates separated by spaces. + /// + /// string array of contextual predicates. + public virtual string[] GetContext(string input) + { + return input.Split(' '); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/BasicEventReader.cs b/BotSharp.MachineLearning/Entropy/BasicEventReader.cs new file mode 100644 index 00000000..6df6ea49 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/BasicEventReader.cs @@ -0,0 +1,125 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the BasicEventStream.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// An object which can deliver a stream of training events assuming + /// that each event is represented as a space separated list containing + /// all the contextual predicates, with the last item being the + /// outcome, e.g.: + /// + ///

cp_1 cp_2 ... cp_n outcome

+ /// + public class BasicEventReader : ITrainingEventReader + { + private IContextGenerator mContext; + private ITrainingDataReader mDataReader; + private TrainingEvent mNextEvent; + + /// + /// Constructor sets up the training event reader based on a stream of training data. + /// + /// + /// Stream of training data. + /// + public BasicEventReader(ITrainingDataReader dataReader) + { + mContext = new BasicContextGenerator(); + + mDataReader = dataReader; + if (mDataReader.HasNext()) + { + mNextEvent = CreateEvent(mDataReader.NextToken()); + } + } + + /// + /// Returns the next Event object held in this EventReader. Each call to ReadNextEvent advances the EventReader. + /// + /// + /// the Event object which is next in this EventReader + /// + public virtual TrainingEvent ReadNextEvent() + { + while (mNextEvent == null && mDataReader.HasNext()) + { + mNextEvent = CreateEvent(mDataReader.NextToken()); + } + + TrainingEvent currentEvent = mNextEvent; + if (mDataReader.HasNext()) + { + mNextEvent = CreateEvent(mDataReader.NextToken()); + } + else + { + mNextEvent = null; + } + return currentEvent; + } + + /// + /// Test whether there are any Events remaining in this EventReader. + /// + /// + /// true if this EventReader has more Events + /// + public virtual bool HasNext() + { + while (mNextEvent == null && mDataReader.HasNext()) + { + mNextEvent = CreateEvent(mDataReader.NextToken()); + } + return mNextEvent != null; + } + + private TrainingEvent CreateEvent(string observation) + { + int lastSpace = observation.LastIndexOf((char)' '); + if (lastSpace == -1) + { + return null; + } + else + { + return new TrainingEvent(observation.Substring(lastSpace + 1), mContext.GetContext(observation.Substring(0, (lastSpace) - (0)))); + } + } + } +} + diff --git a/BotSharp.MachineLearning/Entropy/ComparableEvent.cs b/BotSharp.MachineLearning/Entropy/ComparableEvent.cs new file mode 100644 index 00000000..7962c288 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/ComparableEvent.cs @@ -0,0 +1,220 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the ComparableEvent.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Text; + +namespace BotSharp.MachineLearning +{ + /// + /// A Maximum Entropy event representation which we can use to sort based on the + /// predicates indexes contained in the events. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on ComparableEvent.java, $Revision: 1.2 $, $Date: 2001/12/27 19:20:26 $ + /// + public class ComparableEvent : IComparable + { + private int mOutcome; + private int[] mPredicateIndexes ; + private int mSeenCount = 1; + + /// + /// The outcome ID of this event. + /// + public int Outcome + { + get + { + return mOutcome; + } + set + { + mOutcome = value; + } + } + + /// + /// Returns an array containing the indexes of the predicates in this event. + /// + /// + /// Integer array of predicate indexes. + /// + public int[] GetPredicateIndexes() + { + return mPredicateIndexes; + } + + /// + /// Sets the array containing the indices of the predicates in this event. + /// + /// + /// Integer array of predicate indexes. + /// + public void SetPredicateIndexes(int[] predicateIndexes) + { + mPredicateIndexes = predicateIndexes; + } + + /// + /// The number of times this event + /// has been seen. + /// + public int SeenCount + { + get + { + return mSeenCount; + } + set + { + mSeenCount = value; + } + } + + /// + /// Constructor for the ComparableEvent. + /// + /// + /// The ID of the outcome for this event. + /// + /// + /// Array of indexes for the predicates in this event. + /// + public ComparableEvent(int outcome, int[] predicateIndexes) + { + mOutcome = outcome; + System.Array.Sort(predicateIndexes); + mPredicateIndexes = predicateIndexes; + } + + /// + /// Implementation of the IComparable interface. + /// + /// + /// ComparableEvent to compare this event to. + /// + /// + /// A value indicating if the compared object is smaller, greater or the same as this event. + /// + public virtual int CompareTo(ComparableEvent eventToCompare) + { + if (mOutcome < eventToCompare.Outcome) + { + return - 1; + } + else if (mOutcome > eventToCompare.Outcome) + { + return 1; + } + + int smallerLength = (mPredicateIndexes .Length > eventToCompare.GetPredicateIndexes().Length ? eventToCompare.GetPredicateIndexes().Length : GetPredicateIndexes().Length); + + for (int currentIndex = 0; currentIndex < smallerLength; currentIndex++) + { + if (mPredicateIndexes [currentIndex] < eventToCompare.GetPredicateIndexes()[currentIndex]) + { + return - 1; + } + else if (mPredicateIndexes [currentIndex] > eventToCompare.GetPredicateIndexes()[currentIndex]) + { + return 1; + } + } + + if (mPredicateIndexes .Length < eventToCompare.GetPredicateIndexes().Length) + { + return - 1; + } + else if (mPredicateIndexes .Length > eventToCompare.GetPredicateIndexes().Length) + { + return 1; + } + + return 0; + } + + /// + /// Tests if this event is equal to another object. + /// + /// + /// Object to test against. + /// + /// + /// True if the objects are equal. + /// + public override bool Equals (object o) + { + if (!(o is ComparableEvent)) + { + return false; + } + return (this.CompareTo(o as ComparableEvent)== 0); + } + + /// + /// Provides a hashcode for storing events in a dictionary or hashtable. + /// + /// + /// A hashcode value. + /// + public override int GetHashCode() + { + return this.ToString().GetHashCode(); + } + + /// + /// Override to provide a succint summary of the ComparableEvent object. + /// + /// + /// string representation of the ComparableEvent object. + /// + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + for (int currentIndex = 0; currentIndex < mPredicateIndexes.Length; currentIndex++) + { + stringBuilder.Append(" ").Append(mPredicateIndexes [currentIndex]); + } + return stringBuilder.ToString(); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/GisModel.cs b/BotSharp.MachineLearning/Entropy/GisModel.cs new file mode 100644 index 00000000..f43423e3 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/GisModel.cs @@ -0,0 +1,308 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the GISModel.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.MachineLearning +{ + /// + /// A maximum entropy model which has been trained using the Generalized + /// Iterative Scaling procedure. + /// + /// + /// Tom Morton and Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on GISModel.java, $Revision: 1.13 $, $Date: 2004/06/11 20:51:44 $ + /// + public sealed class GisModel : IMaximumEntropyModel + { + + private readonly IO.IGisModelReader _reader; + + private readonly string[] _outcomeNames; + + private readonly int _outcomeCount; + private readonly double _initialProbability; + private readonly double _correctionConstantInverse; + + private readonly int[] _featureCounts; + + /// + /// Constructor for a maximum entropy model trained using the + /// Generalized Iterative Scaling procedure. + /// + /// + /// A reader providing the data for the model. + /// + public GisModel(IO.IGisModelReader reader) + { + this._reader = reader; + _outcomeNames = reader.GetOutcomeLabels(); + CorrectionConstant = reader.CorrectionConstant; + CorrectionParameter = reader.CorrectionParameter; + + _outcomeCount = _outcomeNames.Length; + _initialProbability = Math.Log(1.0 / _outcomeCount); + _correctionConstantInverse = 1.0 / CorrectionConstant; + _featureCounts = new int[_outcomeCount]; + } + + // implementation of IMaxentModel ------- + + /// + /// Returns the number of outcomes for this model. + /// + /// + /// The number of outcomes. + /// + public int OutcomeCount + { + get + { + return (_outcomeCount); + } + } + + /// + /// Evaluates a context. + /// + /// + /// A list of string names of the contextual predicates + /// which are to be evaluated together. + /// + /// + /// An array of the probabilities for each of the different + /// outcomes, all of which sum to 1. + /// + public double[] Evaluate(string[] context) + { + return Evaluate(context, new double[_outcomeCount]); + } + + /// + /// Use this model to evaluate a context and return an array of the + /// likelihood of each outcome given that context. + /// + /// + /// The names of the predicates which have been observed at + /// the present decision point. + /// + /// + /// This is where the distribution is stored. + /// + /// + /// The normalized probabilities for the outcomes given the + /// context. The indexes of the double[] are the outcome + /// ids, and the actual string representation of the + /// outcomes can be obtained from the method + /// GetOutcome(int outcomeIndex). + /// + public double[] Evaluate(string[] context, double[] outcomeSums) + { + for (int outcomeIndex = 0; outcomeIndex < _outcomeCount; outcomeIndex++) + { + outcomeSums[outcomeIndex] = _initialProbability; + _featureCounts[outcomeIndex] = 0; + } + + foreach (string con in context) + { + _reader.GetPredicateData(con, _featureCounts, outcomeSums); + } + + double normal = 0.0; + for (int outcomeIndex = 0;outcomeIndex < _outcomeCount; outcomeIndex++) + { + outcomeSums[outcomeIndex] = Math.Exp((outcomeSums[outcomeIndex] * _correctionConstantInverse) + ((1.0 - (_featureCounts[outcomeIndex] / CorrectionConstant)) * CorrectionParameter)); + normal += outcomeSums[outcomeIndex]; + } + + for (int outcomeIndex = 0; outcomeIndex < _outcomeCount;outcomeIndex++) + { + outcomeSums[outcomeIndex] /= normal; + } + return outcomeSums; + } + + /// + /// Return the name of the outcome corresponding to the highest likelihood + /// in the parameter outcomes. + /// + /// + /// A double[] as returned by the Evaluate(string[] context) + /// method. + /// + /// + /// The name of the most likely outcome. + /// + public string GetBestOutcome(double[] outcomes) + { + int bestOutcomeIndex = 0; + for (int currentOutcome = 1; currentOutcome < outcomes.Length; currentOutcome++) + if (outcomes[currentOutcome] > outcomes[bestOutcomeIndex]) + { + bestOutcomeIndex = currentOutcome; + } + return _outcomeNames[bestOutcomeIndex]; + } + + /// + /// Return a string matching all the outcome names with all the + /// probabilities produced by the Evaluate(string[] context) + /// method. + /// + /// + /// A double[] as returned by the + /// eval(string[] context) + /// method. + /// + /// + /// string containing outcome names paired with the normalized + /// probability (contained in the double[] outcomes) + /// for each one. + /// + public string GetAllOutcomes(double[] outcomes) + { + if (outcomes.Length != _outcomeNames.Length) + { + throw new ArgumentException("The double array sent as a parameter to GisModel.GetAllOutcomes() must not have been produced by this model."); + } + else + { + var outcomeInfo = new StringBuilder(outcomes.Length * 2); + outcomeInfo.Append(_outcomeNames[0]).Append("[").Append(outcomes[0].ToString("0.0000", System.Globalization.CultureInfo.CurrentCulture)).Append("]"); + for (int currentOutcome = 1; currentOutcome < outcomes.Length; currentOutcome++) + { + outcomeInfo.Append(" ").Append(_outcomeNames[currentOutcome]).Append("[").Append(outcomes[currentOutcome].ToString("0.0000", System.Globalization.CultureInfo.CurrentCulture)).Append("]"); + } + return outcomeInfo.ToString(); + } + } + + /// + /// Return the name of an outcome corresponding to an integer ID value. + /// + /// + /// An outcome ID. + /// + /// + /// The name of the outcome associated with that ID. + /// + public string GetOutcomeName(int outcomeIndex) + { + return _outcomeNames[outcomeIndex]; + } + + /// + /// Gets the index associated with the string name of the given outcome. + /// + /// + /// the string name of the outcome for which the + /// index is desired + /// + /// + /// the index if the given outcome label exists for this + /// model, -1 if it does not. + /// + public int GetOutcomeIndex(string outcome) + { + for (int iCurrentOutcomeName = 0; iCurrentOutcomeName < _outcomeNames.Length; iCurrentOutcomeName++) + { + if (_outcomeNames[iCurrentOutcomeName] == outcome) + { + return iCurrentOutcomeName; + } + } + return - 1; + } + + /// + /// Provides the predicates data structure which is part of the encoding of the maxent model + /// information. This method will usually only be needed by + /// GisModelWriters. + /// + /// + /// Dictionary containing PatternedPredicate objects. + /// + public Dictionary GetPredicates() + { + return _reader.GetPredicates(); + } + + /// + /// Provides the list of outcome patterns used by the predicates. This method will usually + /// only be needed by GisModelWriters. + /// + /// + /// Array of outcome patterns. + /// + public int[][] GetOutcomePatterns() + { + return _reader.GetOutcomePatterns(); + } + + /// + /// Provides the outcome names data structure which is part of the encoding of the maxent model + /// information. This method will usually only be needed by + /// GisModelWriters. + /// + /// + /// Array containing the outcome names. + /// + public string[] GetOutcomeNames() + { + return _outcomeNames; + } + + /// + /// Provides the model's correction constant. + /// This property will usually only be needed by GisModelWriters. + /// + public int CorrectionConstant { get; private set; } + + /// + /// Provides the model's correction parameter. + /// This property will usually only be needed by GisModelWriters. + /// + public double CorrectionParameter { get; private set; } + + } +} diff --git a/BotSharp.MachineLearning/Entropy/GisTrainer.cs b/BotSharp.MachineLearning/Entropy/GisTrainer.cs new file mode 100644 index 00000000..e5ad29b4 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/GisTrainer.cs @@ -0,0 +1,886 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the GISTrainer.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace BotSharp.MachineLearning +{ + /// + /// An implementation of Generalized Iterative Scaling. The reference paper + /// for this implementation was Adwait Ratnaparkhi's tech report at the + /// University of Pennsylvania's Institute for Research in Cognitive Science, + /// and is available at ftp://ftp.cis.upenn.edu/pub/ircs/tr/97-08.ps.Z. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J, Northedge + /// + /// + /// based on GISTrainer.java, $Revision: 1.15 $, $Date: 2004/06/14 20:52:41 $ + /// + public class GisTrainer : IO.IGisModelReader + { + private int mTokenCount; // # of event tokens + private int mPredicateCount; // # of predicates + private int mOutcomeCount; // # of mOutcomes + private int mTokenID; // global index variable for Tokens + private int mPredicateId; // global index variable for Predicates + private int mOutcomeId; // global index variable for Outcomes + + // records the array of predicates seen in each event + private int[][] mContexts; + + // records the array of outcomes seen in each event + private int[] mOutcomes; + + // records the num of times an event has been seen, paired to + // int[][] mContexts + private int[] mNumTimesEventsSeen; + + // stores the string names of the outcomes. The GIS only tracks outcomes + // as ints, and so this array is needed to save the model to disk and + // thereby allow users to know what the outcome was in human + // understandable terms. + private string[] mOutcomeLabels; + + // stores the string names of the predicates. The GIS only tracks + // predicates as ints, and so this array is needed to save the model to + // disk and thereby allow users to know what the outcome was in human + // understandable terms. + private string[] mPredicateLabels; + + // stores the observed expections of each of the events + private double[][] mObservedExpections; + + // stores the estimated parameter value of each predicate during iteration + private double[][] mParameters; + + // Stores the expected values of the features based on the current models + private double[][] mModelExpections; + + //The maximum number of features fired in an event. Usually referred to as C. + private int mMaximumFeatureCount; + + // stores inverse of constant, 1/C. + private double mMaximumFeatureCountInverse; + + // the correction parameter of the model + private double mCorrectionParameter; + + // observed expectation of correction feature + private double mCorrectionFeatureObservedExpectation; + + // a global variable to help compute the amount to modify the correction + // parameter + private double mCorrectionFeatureModifier; + + private const double mNearZero = 0.01; + private const double mLLThreshold = 0.0001; + + // Stores the output of the current model on a single event durring + // training. This will be reset for every event for every iteration. + private double[] mModelDistribution; + + // Stores the number of features that get fired per event + private int[] mFeatureCounts; + + // initial probability for all outcomes. + private double mInitialProbability; + + private Dictionary mPredicates; + private int[][] mOutcomePatterns; + + // smoothing algorithm (unused) -------- + +// internal class UpdateParametersWithSmoothingProcedure : Trove.IIntDoubleProcedure +// { + +// private double mdSigma = 2.0; + +// public UpdateParametersWithSmoothingProcedure(GisTrainer enclosingInstance) +// { +// moEnclosingInstance = enclosingInstance; +// } +// +// private GisTrainer moEnclosingInstance; +// +// public virtual bool Execute(int outcomeID, double input) +// { +// double x = 0.0; +// double x0 = 0.0; +// double tmp; +// double f; +// double fp; +// for (int i = 0; i < 50; i++) +// { +// // check what domain these parameters are in +// tmp = moEnclosingInstance.maoModelExpections[moEnclosingInstance.miPredicateID][outcomeID] * System.Math.Exp(moEnclosingInstance.miConstant * x0); +// f = tmp + (input + x0) / moEnclosingInstance.mdSigma - moEnclosingInstance.maoObservedExpections[moEnclosingInstance.miPredicateID][outcomeID]; +// fp = tmp * moEnclosingInstance.miConstant + 1 / moEnclosingInstance.mdSigma; +// if (fp == 0) +// { +// break; +// } +// x = x0 - f / fp; +// if (System.Math.Abs(x - x0) < 0.000001) +// { +// x0 = x; +// break; +// } +// x0 = x; +// } +// moEnclosingInstance.maoParameters[moEnclosingInstance.miPredicateID].Put(outcomeID, input + x0); +// return true; +// } +// } + + + // training progress event ----------- + + /// + /// Used to provide informational messages regarding the + /// progress of the training algorithm. + /// + public event TrainingProgressEventHandler TrainingProgress; + + /// + /// Used to raise events providing messages with information + /// about training progress. + /// + /// + /// Contains the message with information about the progress of + /// the training algorithm. + /// + protected virtual void OnTrainingProgress(TrainingProgressEventArgs e) + { + if (TrainingProgress != null) + { + TrainingProgress(this, e); + } + } + + private void NotifyProgress(string message) + { + OnTrainingProgress(new TrainingProgressEventArgs(message)); + } + + + // training options -------------- + + /// + /// Sets whether this trainer will use smoothing while training the model. + /// This can improve model accuracy, though training will potentially take + /// longer and use more memory. Model size will also be larger. + /// + /// + /// Initial testing indicates improvements for models built on small data sets and + /// few outcomes, but performance degradation for those with large data + /// sets and lots of outcomes. + /// + public bool Smoothing { get; set; } + + /// + /// Sets whether this trainer will use slack parameters while training the model. + /// + public bool UseSlackParameter { get; set; } + + /// + /// If smoothing is in use, this value indicates the "number" of + /// times we want the trainer to imagine that it saw a feature that it + /// actually didn't see. Defaulted to 0.1. + /// + public double SmoothingObservation { get; set; } + + /// + /// Creates a new GisTrainer instance. + /// + public GisTrainer() + { + Smoothing = false; + UseSlackParameter = false; + SmoothingObservation = 0.1; + } + + /// + /// Creates a new GisTrainer instance. + /// + /// + /// Sets whether this trainer will use slack parameters while training the model. + /// + public GisTrainer(bool useSlackParameter) + { + Smoothing = false; + UseSlackParameter = useSlackParameter; + SmoothingObservation = 0.1; + } + + /// + /// Creates a new GisTrainer instance. + /// + /// + /// If smoothing is in use, this value indicates the "number" of + /// times we want the trainer to imagine that it saw a feature that it + /// actually didn't see. Defaulted to 0.1. + /// + public GisTrainer(double smoothingObservation) + { + Smoothing = true; + UseSlackParameter = false; + SmoothingObservation = smoothingObservation; + } + + /// + /// Creates a new GisTrainer instance. + /// + /// + /// Sets whether this trainer will use slack parameters while training the model. + /// + /// + /// If smoothing is in use, this value indicates the "number" of + /// times we want the trainer to imagine that it saw a feature that it + /// actually didn't see. Defaulted to 0.1. + /// + public GisTrainer(bool useSlackParameter, double smoothingObservation) + { + Smoothing = true; + UseSlackParameter = useSlackParameter; + SmoothingObservation = smoothingObservation; + } + + + // alternative TrainModel signatures -------------- + + /// + /// Train a model using the GIS algorithm. + /// + /// + /// The ITrainingEventReader holding the data on which this model + /// will be trained. + /// + public virtual void TrainModel(ITrainingEventReader eventReader) + { + TrainModel(eventReader, 100, 0); + } + + /// + /// Train a model using the GIS algorithm. + /// + /// + /// The ITrainingEventReader holding the data on which this model will be trained + /// + /// The number of GIS iterations to perform + /// + /// The number of times a predicate must be seen in order + /// to be relevant for training. + /// + public virtual void TrainModel(ITrainingEventReader eventReader, int iterations, int cutoff) + { + TrainModel(iterations, new OnePassDataIndexer(eventReader, cutoff)); + } + + + // training algorithm ----------------------------- + + /// + /// Train a model using the GIS algorithm. + /// + /// + /// The number of GIS iterations to perform. + /// + /// + /// The data indexer used to compress events in memory. + /// + public virtual void TrainModel(int iterations, ITrainingDataIndexer dataIndexer) + { + int[] outcomeList; + + //incorporate all of the needed info + NotifyProgress("Incorporating indexed data for training..."); + mContexts = dataIndexer.GetContexts(); + mOutcomes = dataIndexer.GetOutcomeList(); + mNumTimesEventsSeen = dataIndexer.GetNumTimesEventsSeen(); + mTokenCount = mContexts.Length; + + // determine the correction constant and its inverse + mMaximumFeatureCount = mContexts[0].Length; + for (mTokenID = 1; mTokenID < mContexts.Length; mTokenID++) + { + if (mContexts[mTokenID].Length > mMaximumFeatureCount) + { + mMaximumFeatureCount = mContexts[mTokenID].Length; + } + } + mMaximumFeatureCountInverse = 1.0 / mMaximumFeatureCount; + + NotifyProgress("done."); + + mOutcomeLabels = dataIndexer.GetOutcomeLabels(); + outcomeList = dataIndexer.GetOutcomeList(); + mOutcomeCount = mOutcomeLabels.Length; + mInitialProbability = Math.Log(1.0 / mOutcomeCount); + + mPredicateLabels = dataIndexer.GetPredicateLabels(); + mPredicateCount = mPredicateLabels.Length; + + NotifyProgress("\tNumber of Event Tokens: " + mTokenCount); + NotifyProgress("\t Number of Outcomes: " + mOutcomeCount); + NotifyProgress("\t Number of Predicates: " + mPredicateCount); + + // set up feature arrays + var predicateCounts = new int[mPredicateCount][]; + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + predicateCounts[mPredicateId] = new int[mOutcomeCount]; + } + for (mTokenID = 0; mTokenID < mTokenCount; mTokenID++) + { + for (int currentContext = 0; currentContext < mContexts[mTokenID].Length; currentContext++) + { + predicateCounts[mContexts[mTokenID][currentContext]][outcomeList[mTokenID]] += mNumTimesEventsSeen[mTokenID]; + } + } + + // A fake "observation" to cover features which are not detected in + // the data. The default is to assume that we observed "1/10th" of a + // feature during training. + double smoothingObservation = SmoothingObservation; + + // Get the observed expectations of the features. Strictly speaking, + // we should divide the counts by the number of Tokens, but because of + // the way the model's expectations are approximated in the + // implementation, this is cancelled out when we compute the next + // iteration of a parameter, making the extra divisions wasteful. + mOutcomePatterns = new int[mPredicateCount][]; + mParameters = new double[mPredicateCount][]; + mModelExpections = new double[mPredicateCount][]; + mObservedExpections = new double[mPredicateCount][]; + + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + int activeOutcomeCount; + if (Smoothing) + { + activeOutcomeCount = mOutcomeCount; + } + else + { + activeOutcomeCount = 0; + for (mOutcomeId = 0; mOutcomeId < mOutcomeCount; mOutcomeId++) + { + if (predicateCounts[mPredicateId][mOutcomeId] > 0) + { + activeOutcomeCount++; + } + } + } + + mOutcomePatterns[mPredicateId] = new int[activeOutcomeCount]; + mParameters[mPredicateId] = new double[activeOutcomeCount]; + mModelExpections[mPredicateId] = new double[activeOutcomeCount]; + mObservedExpections[mPredicateId] = new double[activeOutcomeCount]; + + int currentOutcome = 0; + for (mOutcomeId = 0; mOutcomeId < mOutcomeCount; mOutcomeId++) + { + if (predicateCounts[mPredicateId][mOutcomeId] > 0) + { + mOutcomePatterns[mPredicateId][currentOutcome] = mOutcomeId; + mObservedExpections[mPredicateId][currentOutcome] = Math.Log(predicateCounts[mPredicateId][mOutcomeId]); + currentOutcome++; + } + else if (Smoothing) + { + mOutcomePatterns[mPredicateId][currentOutcome] = mOutcomeId; + mObservedExpections[mPredicateId][currentOutcome] = Math.Log(smoothingObservation); + currentOutcome++; + } + } + } + + // compute the expected value of correction + if (UseSlackParameter) + { + int correctionFeatureValueSum = 0; + for (mTokenID = 0; mTokenID < mTokenCount; mTokenID++) + { + for (int currentContext = 0; currentContext < mContexts[mTokenID].Length; currentContext++) + { + mPredicateId = mContexts[mTokenID][currentContext]; + + if ((!Smoothing) && predicateCounts[mPredicateId][mOutcomes[mTokenID]] == 0) + { + correctionFeatureValueSum += mNumTimesEventsSeen[mTokenID]; + } + } + correctionFeatureValueSum += (mMaximumFeatureCount - mContexts[mTokenID].Length) * mNumTimesEventsSeen[mTokenID]; + } + if (correctionFeatureValueSum == 0) + { + mCorrectionFeatureObservedExpectation = Math.Log(mNearZero); //nearly zero so log is defined + } + else + { + mCorrectionFeatureObservedExpectation = Math.Log(correctionFeatureValueSum); + } + + mCorrectionParameter = 0.0; + } + + NotifyProgress("...done."); + + mModelDistribution = new double[mOutcomeCount]; + mFeatureCounts = new int[mOutcomeCount]; + + //Find the parameters + NotifyProgress("Computing model parameters..."); + FindParameters(iterations); + + NotifyProgress("Converting to new predicate format..."); + ConvertPredicates(); + + } + + /// + /// Estimate and return the model parameters. + /// + /// + /// Number of iterations to run through. + /// + private void FindParameters(int iterations) + { + double previousLogLikelihood = 0.0; + NotifyProgress("Performing " + iterations + " iterations."); + for (int currentIteration = 1; currentIteration <= iterations; currentIteration++) + { + if (currentIteration < 10) + { + NotifyProgress(" " + currentIteration + ": "); + } + else if (currentIteration < 100) + { + NotifyProgress(" " + currentIteration + ": "); + } + else + { + NotifyProgress(currentIteration + ": "); + } + double currentLogLikelihood = NextIteration(); + if (currentIteration > 1) + { + if (previousLogLikelihood > currentLogLikelihood) + { + throw new SystemException("Model Diverging: loglikelihood decreased"); + } + if (currentLogLikelihood - previousLogLikelihood < mLLThreshold) + { + break; + } + } + previousLogLikelihood = currentLogLikelihood; + } + + // kill a bunch of these big objects now that we don't need them + mObservedExpections = null; + mModelExpections = null; + mNumTimesEventsSeen = null; + mContexts = null; + } + + /// + /// Use this model to evaluate a context and return an array of the + /// likelihood of each outcome given that context. + /// + /// + /// The integers of the predicates which have been + /// observed at the present decision point. + /// + /// + /// The normalized probabilities for the outcomes given the + /// context. The indexes of the double[] are the outcome + /// ids. + /// + protected virtual void Evaluate(int[] context, double[] outcomeSums) + { + for (int outcomeIndex = 0; outcomeIndex < mOutcomeCount; outcomeIndex++) + { + outcomeSums[outcomeIndex] = mInitialProbability; + mFeatureCounts[outcomeIndex] = 0; + } + int[] activeOutcomes; + int outcomeId; + int predicateId; + int currentActiveOutcome; + + for (int currentContext = 0; currentContext < context.Length; currentContext++) + { + predicateId = context[currentContext]; + activeOutcomes = mOutcomePatterns[predicateId]; + for (currentActiveOutcome = 0; currentActiveOutcome < activeOutcomes.Length; currentActiveOutcome++) + { + outcomeId = activeOutcomes[currentActiveOutcome]; + mFeatureCounts[outcomeId]++; + outcomeSums[outcomeId] += mMaximumFeatureCountInverse * mParameters[predicateId][currentActiveOutcome]; + } + } + + double sum = 0.0; + for (int currentOutcomeId = 0; currentOutcomeId < mOutcomeCount; currentOutcomeId++) + { + outcomeSums[currentOutcomeId] = System.Math.Exp(outcomeSums[currentOutcomeId]); + if (UseSlackParameter) + { + outcomeSums[currentOutcomeId] += ((1.0 - ((double) mFeatureCounts[currentOutcomeId] / mMaximumFeatureCount)) * mCorrectionParameter); + } + sum += outcomeSums[currentOutcomeId]; + } + + for (int currentOutcomeId = 0; currentOutcomeId < mOutcomeCount; currentOutcomeId++) + { + outcomeSums[currentOutcomeId] /= sum; + } + } + + /// + /// Compute one iteration of GIS and retutn log-likelihood. + /// + /// The log-likelihood. + private double NextIteration() + { + // compute contribution of p(a|b_i) for each feature and the new + // correction parameter + double logLikelihood = 0.0; + mCorrectionFeatureModifier = 0.0; + int eventCount = 0; + int numCorrect = 0; + int outcomeId; + + for (mTokenID = 0; mTokenID < mTokenCount; mTokenID++) + { + Evaluate(mContexts[mTokenID], mModelDistribution); + for (int currentContext = 0; currentContext < mContexts[mTokenID].Length; currentContext++) + { + mPredicateId = mContexts[mTokenID][currentContext]; + for (int currentActiveOutcome = 0; currentActiveOutcome < mOutcomePatterns[mPredicateId].Length; currentActiveOutcome++) + { + outcomeId = mOutcomePatterns[mPredicateId][currentActiveOutcome]; + mModelExpections[mPredicateId][currentActiveOutcome] += (mModelDistribution[outcomeId] * mNumTimesEventsSeen[mTokenID]); + + if (UseSlackParameter) + { + mCorrectionFeatureModifier += mModelDistribution[mOutcomeId] * mNumTimesEventsSeen[mTokenID]; + } + } + } + + if (UseSlackParameter) + { + mCorrectionFeatureModifier += (mMaximumFeatureCount - mContexts[mTokenID].Length) * mNumTimesEventsSeen[mTokenID]; + } + + logLikelihood += System.Math.Log(mModelDistribution[mOutcomes[mTokenID]]) * mNumTimesEventsSeen[mTokenID]; + eventCount += mNumTimesEventsSeen[mTokenID]; + + //calculation solely for the information messages + int max = 0; + for (mOutcomeId = 1; mOutcomeId < mOutcomeCount; mOutcomeId++) + { + if (mModelDistribution[mOutcomeId] > mModelDistribution[max]) + { + max = mOutcomeId; + } + } + if (max == mOutcomes[mTokenID]) + { + numCorrect += mNumTimesEventsSeen[mTokenID]; + } + } + NotifyProgress("."); + + // compute the new parameter values + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + for (int currentActiveOutcome = 0; currentActiveOutcome < mOutcomePatterns[mPredicateId].Length; currentActiveOutcome++) + { + outcomeId = mOutcomePatterns[mPredicateId][currentActiveOutcome]; + mParameters[mPredicateId][currentActiveOutcome] += (mObservedExpections[mPredicateId][currentActiveOutcome] - Math.Log(mModelExpections[mPredicateId][currentActiveOutcome])); + mModelExpections[mPredicateId][currentActiveOutcome] = 0.0;// re-initialize to 0.0's + } + } + + if (mCorrectionFeatureModifier > 0.0 && UseSlackParameter) + { + mCorrectionParameter += (mCorrectionFeatureObservedExpectation - Math.Log(mCorrectionFeatureModifier)); + } + + NotifyProgress(". logLikelihood=" + logLikelihood + "\t" + ((double) numCorrect / eventCount)); + return (logLikelihood); + } + + /// + /// Convert the predicate data into the outcome pattern / patterned predicate format used by the GIS models. + /// + private void ConvertPredicates() + { + var predicates = new PatternedPredicate[mParameters.Length]; + + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + double[] parameters = mParameters[mPredicateId]; + predicates[mPredicateId] = new PatternedPredicate(mPredicateLabels[mPredicateId], parameters); + } + + var comparer = new OutcomePatternComparer(); + Array.Sort(mOutcomePatterns, predicates, comparer); + + List outcomePatterns = new List(); + int currentPatternId = 0; + int predicatesInPattern = 0; + int[] currentPattern = mOutcomePatterns[0]; + + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + if (comparer.Compare(currentPattern, mOutcomePatterns[mPredicateId]) == 0) + { + predicates[mPredicateId].OutcomePattern = currentPatternId; + predicatesInPattern++; + } + else + { + int[] pattern = new int[currentPattern.Length + 1]; + pattern[0] = predicatesInPattern; + currentPattern.CopyTo(pattern, 1); + outcomePatterns.Add(pattern); + currentPattern = mOutcomePatterns[mPredicateId]; + currentPatternId++; + predicates[mPredicateId].OutcomePattern = currentPatternId; + predicatesInPattern = 1; + } + } + int[] finalPattern = new int[currentPattern.Length + 1]; + finalPattern[0] = predicatesInPattern; + currentPattern.CopyTo(finalPattern, 1); + outcomePatterns.Add(finalPattern); + + mOutcomePatterns = outcomePatterns.ToArray(); + mPredicates = new Dictionary(predicates.Length); + for (mPredicateId = 0; mPredicateId < mPredicateCount; mPredicateId++) + { + mPredicates.Add(predicates[mPredicateId].Name, predicates[mPredicateId]); + } + } + + + // IGisModelReader implementation -------------------- + + /// + /// The correction constant for the model produced as a result of training. + /// + public int CorrectionConstant + { + get + { + return mMaximumFeatureCount; + } + } + + /// + /// The correction parameter for the model produced as a result of training. + /// + public double CorrectionParameter + { + get + { + return mCorrectionParameter; + } + } + + /// + /// Obtains the outcome labels for the model produced as a result of training. + /// + /// + /// Array of outcome labels. + /// + public string[] GetOutcomeLabels() + { + return mOutcomeLabels; + } + + /// + /// Obtains the outcome patterns for the model produced as a result of training. + /// + /// + /// Array of outcome patterns. + /// + public int[][] GetOutcomePatterns() + { + return mOutcomePatterns; + } + + /// + /// Obtains the predicate data for the model produced as a result of training. + /// + /// + /// Dictionary containing PatternedPredicate objects. + /// + public Dictionary GetPredicates() + { + return mPredicates; + } + + /// + /// Returns trained model information for a predicate, given the predicate label. + /// + /// + /// The predicate label to fetch information for. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method increments the count of each outcome that is active in the specified + /// predicate. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method adds the parameter values for each of the active outcomes in the + /// predicate. + /// + public void GetPredicateData(string predicateLabel, int[] featureCounts, double[] outcomeSums) + { + if (mPredicates.ContainsKey(predicateLabel)) + { + PatternedPredicate predicate = mPredicates[predicateLabel]; + if (predicate != null) + { + int[] activeOutcomes = mOutcomePatterns[predicate.OutcomePattern]; + + for (int currentActiveOutcome = 1; currentActiveOutcome < activeOutcomes.Length; currentActiveOutcome++) + { + int outcomeIndex = activeOutcomes[currentActiveOutcome]; + featureCounts[outcomeIndex]++; + outcomeSums[outcomeIndex] += predicate.GetParameter(currentActiveOutcome - 1); + } + } + } + } + + + private class OutcomePatternComparer : IComparer + { + + internal OutcomePatternComparer() + { + } + + /// + /// Compare two outcome patterns and determines which comes first, + /// based on the outcome ids (lower outcome ids first) + /// + /// + /// First outcome pattern to compare. + /// + /// + /// Second outcome pattern to compare. + /// + /// + public virtual int Compare(int[] firstPattern, int[] secondPattern) + { + int smallerLength = (firstPattern.Length > secondPattern.Length ? secondPattern.Length : firstPattern.Length); + + for (int currentOutcome = 0; currentOutcome < smallerLength; currentOutcome++) + { + if (firstPattern[currentOutcome] < secondPattern[currentOutcome]) + { + return - 1; + } + else if (firstPattern[currentOutcome] > secondPattern[currentOutcome]) + { + return 1; + } + } + + if (firstPattern.Length < secondPattern.Length) + { + return - 1; + } + else if (firstPattern.Length > secondPattern.Length) + { + return 1; + } + + return 0; + } + } + } + + /// + /// Event arguments class for training progress events. + /// + public class TrainingProgressEventArgs : EventArgs + { + private string mMessage; + + /// + /// Constructor for the training progress event arguments. + /// + /// + /// Information message about the progress of training. + /// + public TrainingProgressEventArgs(string message) + { + mMessage = message; + } + + /// + /// Information message about the progress of training. + /// + public string Message + { + get + { + return mMessage; + } + } + } + + /// + /// Event handler delegate for the training progress event. + /// + public delegate void TrainingProgressEventHandler(object sender, TrainingProgressEventArgs e); + + +} diff --git a/BotSharp.MachineLearning/Entropy/IContextGenerator.cs b/BotSharp.MachineLearning/Entropy/IContextGenerator.cs new file mode 100644 index 00000000..9431de9c --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IContextGenerator.cs @@ -0,0 +1,71 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the ContextGenerator.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// Generate contexts for maximum entropy decisions. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on ContextGenerator.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public interface IContextGenerator + { + /// + /// Builds up the list of contextual predicates given an object. + /// + string[] GetContext(object input); + } + + /// + /// Generate contexts for maximum entropy decisions. + /// + public interface IContextGenerator + { + /// + /// Builds up the list of contextual predicates given an object of type T. + /// + string[] GetContext(T input); + } + +} diff --git a/BotSharp.MachineLearning/Entropy/IMaximumEntropyModel.cs b/BotSharp.MachineLearning/Entropy/IMaximumEntropyModel.cs new file mode 100644 index 00000000..952f4126 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IMaximumEntropyModel.cs @@ -0,0 +1,151 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the MaxentModel.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// Interface for maximum entropy models. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on MaxentModel.java, $Revision: 1.4 $, $Date: 2003/12/09 23:13:53 $ + /// + public interface IMaximumEntropyModel + { + /// + /// Returns the number of outcomes for this model. + /// + /// + /// The number of outcomes. + /// + int OutcomeCount + { + get; + } + + /// + /// Evaluates a context. + /// + /// + /// A list of string names of the contextual predicates + /// which are to be evaluated together. + /// + /// + /// An array of the probabilities for each of the different + /// outcomes, all of which sum to 1. + /// + double[] Evaluate(string[] context); + + /// + /// Evaluates a context. + /// + /// + /// A list of string names of the contextual predicates + /// which are to be evaluated together. + /// + /// + /// An array which is populated with the probabilities for each of the different + /// outcomes, all of which sum to 1. + /// + /// + /// an array of the probabilities for each of the different + /// outcomes, all of which sum to 1. The probabilities array is returned if it is appropiately sized. + /// + double[] Evaluate(string[] context, double[] probabilities); + + /// + /// Simple function to return the outcome associated with the index + /// containing the highest probability in the double[]. + /// + /// + /// A double[] as returned by the + /// Evaluate(string[] context) + /// method. + /// + /// + /// the string name of the best outcome + /// + string GetBestOutcome(double[] outcomes); + + /// + /// Return a string matching all the outcome names with all the + /// probabilities produced by the eval(string[] + /// context) method. + /// + /// + /// A double[] as returned by the + /// eval(string[] context) + /// method. + /// + /// + /// string containing outcome names paired with the normalized + /// probability (contained in the double[] ocs) + /// for each one. + /// + string GetAllOutcomes(double[] outcomes); + + /// + /// Gets the string name of the outcome associated with the supplied index + /// + /// + /// the index for which the name of the associated outcome is desired. + /// + /// + /// the string name of the outcome + /// + string GetOutcomeName(int index); + + /// + /// Gets the index associated with the string name of the given + /// outcome. + /// + /// + /// the string name of the outcome for which the + /// index is desired + /// + /// + /// the index if the given outcome label exists for this + /// model, -1 if it does not. + /// + int GetOutcomeIndex(string outcome); + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/BinaryGisModelReader.cs b/BotSharp.MachineLearning/Entropy/IO/BinaryGisModelReader.cs new file mode 100644 index 00000000..a34bae18 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/BinaryGisModelReader.cs @@ -0,0 +1,175 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the BinaryGISModelReader.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// A reader for GIS models stored in a binary format. This format is not the one + /// used by the java version of MaxEnt. + /// It has two main differences, designed for performance when loading the data + /// from file: first, it uses big endian data values, which is native for C#, and secondly it + /// encodes the outcome patterns and values in a more efficient manner. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on BinaryGISModelReader.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class BinaryGisModelReader : GisModelReader + { + private readonly Stream _input; + private readonly byte[] _buffer; + private int _stringLength = 0; + private readonly Encoding _encoding = Encoding.UTF8; + + /// + /// Constructor which directly instantiates the Stream containing + /// the model contents. + /// + /// + /// The Stream containing the model information. + /// + public BinaryGisModelReader(Stream dataInputStream) + { + using (_input = dataInputStream) + { + _buffer = new byte[256]; + base.ReadModel(); + } + } + + /// + /// Constructor which takes a filename and creates a reader for it. + /// + /// + /// The full path and name of the file in which the model is stored. + /// + public BinaryGisModelReader(string fileName) + { + using (_input = new FileStream(fileName, FileMode.Open, FileAccess.Read)) + { + _buffer = new byte[256]; + base.ReadModel(); + } + } + + /// + /// Reads a 32-bit signed integer from the model file. + /// + protected override int ReadInt32() + { + _input.Read(_buffer, 0, 4); + return BitConverter.ToInt32(_buffer, 0); + } + + /// + /// Reads a double-precision floating point number from the model file. + /// + protected override double ReadDouble() + { + _input.Read(_buffer, 0, 8); + return BitConverter.ToDouble(_buffer, 0); + } + + /// + /// Reads a UTF-8 encoded string from the model file. + /// + protected override string ReadString() + { + _stringLength = _input.ReadByte(); + _input.Read(_buffer, 0, _stringLength); + return _encoding.GetString(_buffer, 0, _stringLength); + } + + /// + /// Reads the predicate data from the file in a more efficient format to that implemented by + /// GisModelReader. + /// + /// + /// Jagged 2-dimensional array of integers that will contain the outcome patterns for the model + /// after this method is called. + /// + /// + /// Dictionary that will contain the predicate information for the model + /// after this method is called. + /// + protected override void ReadPredicates(out int[][] outcomePatterns, out Dictionary predicates) + { + //read from the model how many outcome patterns there are + int outcomePatternCount = ReadInt32(); + outcomePatterns = new int[outcomePatternCount][]; + //read from the model how many predicates there are + predicates = new Dictionary(ReadInt32()); + + //for each outcome pattern in the model + for (int currentOutcomePattern = 0; currentOutcomePattern < outcomePatternCount; currentOutcomePattern++) + { + //read the number of outcomes in this pattern. This number is 1 greater than the real number of outcomes + //in the pattern, because the 0th value contains the number of predicates that use this pattern. + var currentOutcomePatternLength = ReadInt32(); + outcomePatterns[currentOutcomePattern] = new int[currentOutcomePatternLength]; + //read in the outcomes for this pattern + for (int currentOutcome = 0; currentOutcome + /// A writer for GIS models that saves models in a binary format. This format is not the one + /// used by the java version of MaxEnt. + /// It has two main differences, designed for performance when loading the data + /// from file: first, it uses big endian data values, which is native for C#, and secondly it + /// encodes the outcome patterns and values in a more efficient manner. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on BinaryGISModelWriter.java $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class BinaryGisModelWriter : GisModelWriter + { + private Stream _output; + private byte[] _buffer = new byte[7]; + private readonly System.Text.Encoding _encoding = System.Text.Encoding.UTF8; + + /// + /// Default constructor. + /// + public BinaryGisModelWriter(){} + + /// + /// Takes a GIS model and a file and + /// writes the model to that file. + /// + /// + /// The GisModel which is to be persisted. + /// + /// + /// The full path and name of the file in which the model is to be persisted. + /// + public void Persist(GisModel model, string fileName) + { + using (_output = new FileStream(fileName, FileMode.Create)) + { + base.Persist(model); + } + } + + /// + /// Takes a GIS model and a Stream and + /// writes the model to that Stream. + /// + /// + /// The GIS model which is to be persisted. + /// + /// + /// The Stream which will be used to persist the model. + /// + public void Persist(GisModel model, Stream dataOutputStream) + { + using (_output = dataOutputStream) + { + base.Persist(model); + } + } + + /// + /// Writes a UTF-8 encoded string to the model file. + /// + /// + /// The string data to be persisted. + /// + protected override void WriteString(string data) + { + _output.WriteByte((byte)_encoding.GetByteCount(data)); + _output.Write(_encoding.GetBytes(data), 0, _encoding.GetByteCount(data)); + } + + /// + /// Writes a 32-bit signed integer to the model file. + /// + /// + /// The integer data to be persisted. + /// + protected override void WriteInt32(int data) + { + _buffer = BitConverter.GetBytes(data); + _output.Write(_buffer, 0, 4); + } + + /// + /// Writes a double-precision floating point number to the model file. + /// + /// + /// The floating point data to be persisted. + /// + protected override void WriteDouble(double data) + { + _buffer = BitConverter.GetBytes(data); + _output.Write(_buffer, 0, 8); + } + + /// + /// Writes the predicate data to the file in a more efficient format to that implemented by + /// GisModelWriter. + /// + /// + /// The GIS model containing the predicate data to be persisted. + /// + protected override void WritePredicates(GisModel model) + { + int[][] outcomePatterns = model.GetOutcomePatterns(); + PatternedPredicate[] predicates = GetPredicates(); + + //write the number of outcome patterns + WriteInt32(outcomePatterns.Length); + + //write the number of predicates + WriteInt32(predicates.Length); + + int currentPredicate = 0; + + for (int currentOutcomePattern = 0; currentOutcomePattern < outcomePatterns.Length; currentOutcomePattern++) + { + //write how many outcomes in this pattern + WriteInt32(outcomePatterns[currentOutcomePattern].Length); + + //write the outcomes in this pattern (the first value contains the number of predicates in the pattern + //rather than an outcome) + for (int currentOutcome = 0; currentOutcome < outcomePatterns[currentOutcomePattern].Length; currentOutcome++) + { + WriteInt32(outcomePatterns[currentOutcomePattern][currentOutcome]); + } + + //write predicates for this pattern + while (currentPredicate < predicates.Length && predicates[currentPredicate].OutcomePattern == currentOutcomePattern) + { + WriteString(predicates[currentPredicate].Name); + for (int currentParameter = 0; currentParameter < predicates[currentPredicate].ParameterCount; currentParameter++) + { + WriteDouble(predicates[currentPredicate].GetParameter(currentParameter)); + } + currentPredicate++; + } + } + } + + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/GisModelReader.cs b/BotSharp.MachineLearning/Entropy/IO/GisModelReader.cs new file mode 100644 index 00000000..fbd7eb50 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/GisModelReader.cs @@ -0,0 +1,347 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the GISModelReader.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections.Generic; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// Abstract parent class for readers of GIS models. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on GISModelReader.java, $Revision: 1.5 $, $Date: 2004/06/11 20:51:36 $ + /// + public abstract class GisModelReader : IGisModelReader + { + private char[] _spaces; + private int _correctionConstant; + private double _correctionParameter; + private string[] _outcomeLabels; + private int[][] _outcomePatterns; + private int _predicateCount; + private Dictionary _predicates; + + /// + /// The number of predicates contained in the model. + /// + protected int PredicateCount + { + get + { + return _predicateCount; + } + } + + /// + /// Retrieve a model from disk. + /// + ///

This method delegates to worker methods for each part of this + /// sequence. If you are creating a reader that conforms largely to this + /// sequence but varies at one or more points, override the relevant worker + /// method(s) to achieve the required format.

+ /// + ///

If you are creating a reader for a format which does not follow this + /// sequence at all, override this method and ignore the + /// other ReadX methods provided in this abstract class.

+ /// + /// + /// Thie method assumes that models are saved in the + /// following sequence: + /// + ///

GIS (model type identifier)

+ ///

1. the correction constant (int)

+ ///

2. the correction constant parameter (double)

+ ///

3. outcomes

+ ///

3a. number of outcomes (int)

+ ///

3b. outcome names (string array - length specified in 3a)

+ ///

4. predicates

+ ///

4a. outcome patterns

+ ///

4ai. number of outcome patterns (int)

+ ///

4aii. outcome pattern values (each stored in a space delimited string)

+ ///

4b. predicate labels

+ ///

4bi. number of predicates (int)

+ ///

4bii. predicate names (string array - length specified in 4bi)

+ ///

4c. predicate parameters (double values)

+ /// + protected virtual void ReadModel() + { + _spaces = new char[] {' '}; //cached constant to improve performance + CheckModelType(); + _correctionConstant = ReadCorrectionConstant(); + _correctionParameter = ReadCorrectionParameter(); + _outcomeLabels = ReadOutcomes(); + ReadPredicates(out _outcomePatterns, out _predicates); + } + + /// + /// Checks the model file being read from begins with the sequence of characters + /// "GIS". + /// + protected virtual void CheckModelType() + { + string modelType = ReadString(); + if (modelType != "GIS") + { + throw new ApplicationException("Error: attempting to load a " + modelType + " model as a GIS model." + " You should expect problems."); + } + } + + /// + /// Reads the correction constant from the model file. + /// + protected virtual int ReadCorrectionConstant() + { + return ReadInt32(); + } + + /// + /// Reads the correction constant parameter from the model file. + /// + protected virtual double ReadCorrectionParameter() + { + return ReadDouble(); + } + + /// + /// Reads the outcome names from the model file. + /// + protected virtual string[] ReadOutcomes() + { + int outcomeCount = ReadInt32(); + var outcomeLabels = new string[outcomeCount]; + for (int currentLabel = 0; currentLabel < outcomeCount; currentLabel++) + { + outcomeLabels[currentLabel] = ReadString(); + } + return outcomeLabels; + } + + /// + /// Reads the predicate information from the model file, placing the data in two + /// structures - an array of outcome patterns, and a Dictionary of predicates + /// keyed by predicate name. + /// + protected virtual void ReadPredicates(out int[][] outcomePatterns, out Dictionary predicates) + { + outcomePatterns = ReadOutcomePatterns(); + string[] asPredicateLabels = ReadPredicateLabels(); + predicates = ReadParameters(outcomePatterns, asPredicateLabels); + } + + /// + /// Reads the outcome pattern information from the model file. + /// + protected virtual int[][] ReadOutcomePatterns() + { + //get the number of outcome patterns (that is, the number of unique combinations of outcomes in the model) + int outcomePatternCount = ReadInt32(); + //initialize an array of outcome patterns. Each outcome pattern is itself an array of integers + var outcomePatterns = new int[outcomePatternCount][]; + //for each outcome pattern + for (int currentOutcomePattern = 0; currentOutcomePattern < outcomePatternCount; currentOutcomePattern++) + { + //read a space delimited string from the model file containing the information for the integer array. + //The first value in the integer array is the number of predicates related to this outcome pattern; the + //other values make up the outcome IDs for this pattern. + string[] tokens = ReadString().Split(_spaces); + //convert this string to the array of integers required for the pattern + var patternData = new int[tokens.Length]; + for (int currentPatternValue = 0; currentPatternValue < tokens.Length; currentPatternValue++) + { + patternData[currentPatternValue] = int.Parse(tokens[currentPatternValue], System.Globalization.CultureInfo.InvariantCulture); + } + outcomePatterns[currentOutcomePattern] = patternData; + } + return outcomePatterns; + } + + /// + /// Reads the outcome labels from the model file. + /// + protected virtual string[] ReadPredicateLabels() + { + _predicateCount = ReadInt32(); + var predicateLabels = new string[_predicateCount]; + for (int currentPredicate = 0; currentPredicate < _predicateCount; currentPredicate++) + { + predicateLabels[currentPredicate] = ReadString(); + } + return predicateLabels; + } + + /// + /// Reads the predicate parameter information from the model file. + /// + protected virtual Dictionary ReadParameters(int[][] outcomePatterns, string[] predicateLabels) + { + var predicates = new Dictionary(predicateLabels.Length); + int parameterIndex = 0; + + for (int currentOutcomePattern = 0; currentOutcomePattern < outcomePatterns.Length; currentOutcomePattern++) + { + for (int currentOutcomeInfo = 0; currentOutcomeInfo < outcomePatterns[currentOutcomePattern][0]; currentOutcomeInfo++) + { + var parameters = new double[outcomePatterns[currentOutcomePattern].Length - 1]; + for (int currentParameter = 0; currentParameter < outcomePatterns[currentOutcomePattern].Length - 1; currentParameter++) + { + parameters[currentParameter] = ReadDouble(); + } + predicates.Add(predicateLabels[parameterIndex], new PatternedPredicate(currentOutcomePattern, parameters)); + parameterIndex++; + } + } + return predicates; + } + + /// + /// Implement as needed for the format the model is stored in. + /// + protected abstract int ReadInt32(); + + /// + /// Implement as needed for the format the model is stored in. + /// + protected abstract double ReadDouble(); + + /// + /// Implement as needed for the format the model is stored in. + /// + protected abstract string ReadString(); + + /// + /// The model's correction constant. + /// + public int CorrectionConstant + { + get + { + return _correctionConstant; + } + } + + /// + /// The model's correction constant parameter. + /// + public double CorrectionParameter + { + get + { + return _correctionParameter; + } + } + + /// + /// Returns the labels for all the outcomes in the model. + /// + /// + /// string array containing outcome labels. + /// + public string[] GetOutcomeLabels() + { + return _outcomeLabels; + } + + /// + /// Returns the outcome patterns in the model. + /// + /// + /// Array of integer arrays containing the information for + /// each outcome pattern in the model. + /// + public int[][] GetOutcomePatterns() + { + return _outcomePatterns; + } + + /// + /// Returns the predicates in the model. + /// + /// + /// Dictionary containing PatternedPredicate objects keyed + /// by predicate label. + /// + public Dictionary GetPredicates() + { + return _predicates; + } + + /// + /// Returns model information for a predicate, given the predicate label. + /// + /// + /// The predicate label to fetch information for. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method increments the count of each outcome that is active in the specified + /// predicate. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method adds the parameter values for each of the active outcomes in the + /// predicate. + /// + public virtual void GetPredicateData(string predicateLabel, int[] featureCounts, double[] outcomeSums) + { + try + { + if (predicateLabel != null && _predicates.ContainsKey(predicateLabel)) + { + PatternedPredicate predicate = _predicates[predicateLabel]; + int[] activeOutcomes = _outcomePatterns[predicate.OutcomePattern]; + + for (int currentActiveOutcome = 1; currentActiveOutcome < activeOutcomes.Length; currentActiveOutcome++) + { + int outcomeIndex = activeOutcomes[currentActiveOutcome]; + featureCounts[outcomeIndex]++; + outcomeSums[outcomeIndex] += predicate.GetParameter(currentActiveOutcome - 1); + } + } + } + catch (ArgumentNullException ex) + { + throw new ArgumentException(string.Format("Try to find key '{0}' in predicates dictionary ({1} entries)", predicateLabel, _predicates.Count), ex); + } + } + + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/GisModelWriter.cs b/BotSharp.MachineLearning/Entropy/IO/GisModelWriter.cs new file mode 100644 index 00000000..6a969f12 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/GisModelWriter.cs @@ -0,0 +1,313 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the GISModeWriter.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.MachineLearning.IO +{ + /// Abstract parent class for GIS model writers that save data to a single + /// file. It provides the persist method which takes care of the structure of a stored + /// document, and requires an extending class to define precisely how the data should + /// be stored. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on GISModelWriter.java, $Revision: 1.5 $, $Date: 2004/06/11 20:51:36 $ + /// + public abstract class GisModelWriter + { + private PatternedPredicate[] mPredicates; + + /// + /// Implement as needed for the format the model is stored in. + /// + /// + /// string data to be written to storage. + /// + protected abstract void WriteString(string data); + + /// + /// Implement as needed for the format the model is stored in. + /// + /// + /// Integer data to be written to storage. + /// + protected abstract void WriteInt32(int data); + + /// + /// Implement as needed for the format the model is stored in. + /// + /// + /// Double precision floating point data to be written to storage. + /// + protected abstract void WriteDouble(double data); + + /// + /// Obtains a list of the predicates in the model to be written to storage. + /// + /// + /// Array of PatternedPredicate objects containing the predicate data for the model. + /// + protected PatternedPredicate[] GetPredicates() + { + return mPredicates; + } + + /// + /// Sets the list of predicates to be written to storage. + /// + /// + /// Array of PatternedPredicate objects to be persisted. + /// + protected void SetPredicates(PatternedPredicate[] predicates) + { + mPredicates = predicates; + } + + /// + /// Writes the model to persistent storage, using the writeX() methods + /// provided by extending classes. + /// + ///

This method delegates to worker methods for each part of this + /// sequence. If you are creating a writer that conforms largely to this + /// sequence but varies at one or more points, override the relevant worker + /// method(s) to achieve the required format.

+ /// + ///

If you are creating a writer for a format which does not follow this + /// sequence at all, override this method and ignore the + /// other WriteX methods provided in this abstract class.

+ /// + /// + /// GIS model whose data is to be persisted. + /// + protected void Persist(GisModel model) + { + Initialize(model); + WriteModelType("GIS"); + WriteCorrectionConstant(model.CorrectionConstant); + WriteCorrectionParameter(model.CorrectionParameter); + WriteOutcomes(model.GetOutcomeNames()); + WritePredicates(model); + } + + /// + /// Organises the data available in the GIS model into a structure that is easier to + /// persist from. + /// + /// + /// The GIS model to be persisted. + /// + protected virtual void Initialize(GisModel model) + { + //read the predicates from the model + Dictionary predicates = model.GetPredicates(); + //build arrays of predicates and predicate names from the dictionary + mPredicates = new PatternedPredicate[predicates.Count]; + var predicateNames = new string[predicates.Count]; + predicates.Values.CopyTo(mPredicates, 0); + predicates.Keys.CopyTo(predicateNames, 0); + //give each PatternedPredicate in the array the name taken from the dictionary keys + for (int currentPredicate = 0; currentPredicate < predicates.Count; currentPredicate++) + { + mPredicates[currentPredicate].Name = predicateNames[currentPredicate]; + } + //sort the PatternedPredicate array based on the outcome pattern that each predicate uses + Array.Sort(mPredicates, new OutcomePatternIndexComparer()); + } + + /// + /// Writes the model type identifier at the beginning of the file. + /// + /// string identifying the model type. + protected virtual void WriteModelType(string modelType) + { + WriteString(modelType); + } + + /// + /// Writes the value of the correction constant + /// + /// the model's correction constant value. + protected virtual void WriteCorrectionConstant(int correctionConstant) + { + WriteInt32(correctionConstant); + } + + /// + /// Writes the value of the correction constant parameter. + /// + /// the model's correction constant parameter. + protected virtual void WriteCorrectionParameter(double correctionParameter) + { + WriteDouble(correctionParameter); + } + + /// + /// Writes the outcome labels to the file. + /// + /// string array of outcome labels. + protected virtual void WriteOutcomes(string[] outcomeLabels) + { + //write the number of outcomes + WriteInt32(outcomeLabels.Length); + + //write each label + foreach (string label in outcomeLabels) + { + WriteString(label); + } + } + + /// + /// Writes the predicate information to the model file. + /// + /// The GIS model to write the data from. + protected virtual void WritePredicates(GisModel model) + { + WriteOutcomePatterns(model.GetOutcomePatterns()); + WritePredicateNames(); + WriteParameters(); + } + + /// + /// Writes the outcome pattern data to the file. + /// + /// + /// Array of outcome patterns, each an integer array containing + /// the number of predicates using the pattern, and then the list of + /// outcome IDs in the pattern. + /// + protected void WriteOutcomePatterns(int[][] outcomePatterns) + { + //write the number of outcome patterns + WriteInt32(outcomePatterns.Length); + + //for each pattern + foreach (int[] pattern in outcomePatterns) + { + //build a string with the pattern values separated by spaces + var outcomePatternBuilder = new StringBuilder(); + for (int currentOutcome = 0; currentOutcome < pattern.Length; currentOutcome++) + { + if (currentOutcome > 0) + { + outcomePatternBuilder.Append(" "); + } + outcomePatternBuilder.Append(pattern[currentOutcome]); + } + //write the string containing pattern values to the file + WriteString(outcomePatternBuilder.ToString()); + } + } + + /// + /// Write the names of the predicates to the model file. + /// + protected void WritePredicateNames() + { + //write the number of predicates + WriteInt32(mPredicates.Length); + + //for each predicate, write its name to the file + foreach (PatternedPredicate predicate in mPredicates) + { + WriteString(predicate.Name); + } + } + + /// + /// Writes out the parameter values for all the predicates to the model file. + /// + protected void WriteParameters() + { + foreach (PatternedPredicate predicate in mPredicates) + { + for (int currentParameter = 0; currentParameter < predicate.ParameterCount; currentParameter++) + { + WriteDouble(predicate.GetParameter(currentParameter)); + } + } + } + + /// + /// Class to enable sorting PatternedPredicates into order based on the + /// outcome pattern index. + /// + private class OutcomePatternIndexComparer : IComparer + { + + /// + /// Default constructor. + /// + internal OutcomePatternIndexComparer(){} + + /// + /// Implementation of the IComparer interface. + /// Compares two PatternedPredicate objects and returns a value indicating whether + /// one is less than, equal to or greater than the other. + /// + /// + /// First object to compare. + /// + /// + /// Second object to compare. + /// + /// + /// -1 if the first PatternedPredicate has a lower outcome pattern index; + /// 1 if the second PatternedPredicate has a lower outcome pattern index; + /// 0 if they both have the same outcome pattern index. + /// + public virtual int Compare(PatternedPredicate firstPredicate, PatternedPredicate secondPredicate) + { + if (firstPredicate.OutcomePattern < secondPredicate.OutcomePattern) + { + return -1; + } + else if (firstPredicate.OutcomePattern > secondPredicate.OutcomePattern) + { + return 1; + } + return 0; + } + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/IGisModelReader.cs b/BotSharp.MachineLearning/Entropy/IO/IGisModelReader.cs new file mode 100644 index 00000000..a1c022ac --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/IGisModelReader.cs @@ -0,0 +1,87 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file has no equivalent in the java MaxEnt library, because the link +//between GISModel and GISModelReader is implemented differently there. This +//interface is designed so that GIS model reader classes can hold some or all of +//their data in persistent storage rather than in memory. + +using System; +using System.Collections.Generic; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// Interface for readers of GIS models. + /// + public interface IGisModelReader + { + /// + /// Returns the value of the model's correction constant. This property should + /// usually only be accessed by GIS model writer classes via the GisModel class. + /// + int CorrectionConstant + { + get; + } + + /// + /// Returns the value of the model's correction constant parameter. This property should + /// usually only be accessed by GIS model writer classes via the GisModel class. + /// + double CorrectionParameter + { + get; + } + + /// + /// Returns the model's outcome labels as a string array. This method should + /// usually only be accessed by GIS model writer classes via the GisModel class. + /// + string[] GetOutcomeLabels(); + + /// + /// Returns the model's outcome patterns. This method should + /// usually only be accessed by GIS model writer classes via the GisModel class. + /// + int[][] GetOutcomePatterns(); + + /// + /// Returns the model's predicates. This method should + /// usually only be accessed by GIS model writer classes via the GisModel class. + /// + Dictionary GetPredicates(); + + /// + /// Returns model information for a predicate, given the predicate label. + /// + /// + /// The predicate label to fetch information for. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method increments the count of each outcome that is active in the specified + /// predicate. + /// + /// + /// Array to be passed in to the method; it should have a length equal to the number of outcomes + /// in the model. The method adds the parameter values for each of the active outcomes in the + /// predicate. + /// + void GetPredicateData(string predicateLabel, int[] featureCounts, double[] outcomeSums); + + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelReader.cs b/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelReader.cs new file mode 100644 index 00000000..5d577ac2 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelReader.cs @@ -0,0 +1,123 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the BinaryGISModelReader.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// A reader for GIS models stored in the binary format produced by the java version + /// of MaxEnt. This binary format stores data using big-endian values, which means + /// that the C# version must reverse the byte order of each value in turn, making it + /// less efficient. Use only for compatibility with the java MaxEnt library. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on BinaryGISModelReader.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class JavaBinaryGisModelReader : GisModelReader + { + private readonly Stream _input; + private readonly byte[] _buffer; + private int _stringLength = 0; + private readonly System.Text.Encoding _encoding = System.Text.Encoding.UTF8; + + /// + /// Constructor which directly instantiates the Stream containing + /// the model contents. + /// + /// The Stream containing the model information. + /// + public JavaBinaryGisModelReader(Stream dataInputStream) + { + using (_input = dataInputStream) + { + _buffer = new byte[256]; + base.ReadModel(); + } + } + + /// + /// Constructor which takes a filename and creates a reader for it. + /// + /// The full path and name of the file in which the model is stored. + /// + public JavaBinaryGisModelReader(string fileName) + { + using (_input = new FileStream(fileName, FileMode.Open, FileAccess.Read)) + { + _buffer = new byte[256]; + base.ReadModel(); + } + } + + /// + /// Reads a 32-bit signed integer from the model file. + /// + protected override int ReadInt32() + { + _input.Read(_buffer, 0, 4); + Array.Reverse(_buffer, 0, 4); + return BitConverter.ToInt32(_buffer, 0); + } + + /// + /// Reads a double-precision floating point number from the model file. + /// + protected override double ReadDouble() + { + _input.Read(_buffer, 0, 8); + Array.Reverse(_buffer, 0, 8); + return BitConverter.ToDouble(_buffer, 0); + } + + /// + /// Reads a UTF-8 encoded string from the model file. + /// + protected override string ReadString() + { + //read string from binary file with UTF8 encoding + _stringLength = (_input.ReadByte() * 256) + _input.ReadByte(); + _input.Read(_buffer, 0, _stringLength); + return _encoding.GetString(_buffer, 0, _stringLength); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelWriter.cs b/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelWriter.cs new file mode 100644 index 00000000..fe00469b --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/JavaBinaryGisModelWriter.cs @@ -0,0 +1,140 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the BinaryGISModelWriter.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// A writer for GIS models that saves models in the binary format used by the java + /// version of MaxEnt. This binary format stores data using big-endian values, which means + /// that the C# version must reverse the byte order of each value in turn, making it + /// less efficient. Use only for compatibility with the java MaxEnt library. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on BinaryGISModelWriter.java $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class JavaBinaryGisModelWriter : GisModelWriter + { + private Stream mOutput; + private byte[] mBuffer = new byte[7]; + private System.Text.Encoding mEncoding = System.Text.Encoding.UTF8; + + /// + /// Default constructor. + /// + public JavaBinaryGisModelWriter() + { + } + + /// Takes a GisModel and a File and + /// writes the model to that file. + /// + /// The GisModel which is to be persisted. + /// + /// The name of the file in which the model is to be persisted. + /// + public void Persist(GisModel model, string fileName) + { + using (mOutput = new FileStream(fileName, FileMode.Create)) + { + base.Persist(model); + } + } + + /// + /// Takes a GisModel and a Stream and writes the model to that stream. + /// + /// + /// The GIS model which is to be persisted. + /// + /// + /// The Stream which will be used to persist the model. + /// + public void Persist(GisModel model, Stream dataOutputStream) + { + using (mOutput = dataOutputStream) + { + base.Persist(model); + } + } + + /// + /// Writes a UTF-8 encoded string to the model file. + /// + /// /// + /// The string data to be persisted. + /// + protected override void WriteString(string data) + { + mOutput.WriteByte((byte)(mEncoding.GetByteCount(data) / 256)); + mOutput.WriteByte((byte)(mEncoding.GetByteCount(data) % 256)); + mOutput.Write(mEncoding.GetBytes(data), 0, mEncoding.GetByteCount(data)); + } + + /// + /// Writes a 32-bit signed integer to the model file. + /// + /// /// + /// The integer data to be persisted. + /// + protected override void WriteInt32(int data) + { + mBuffer = BitConverter.GetBytes(data); + Array.Reverse(mBuffer); + mOutput.Write(mBuffer, 0, 4); + } + + /// + /// Writes a double-precision floating point number to the model file. + /// + /// /// + /// The floating point data to be persisted. + /// + protected override void WriteDouble(double data) + { + mBuffer = BitConverter.GetBytes(data); + Array.Reverse(mBuffer); + mOutput.Write(mBuffer, 0, 8); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelReader.cs b/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelReader.cs new file mode 100644 index 00000000..af27636c --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelReader.cs @@ -0,0 +1,111 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the PlainTextGISModelReader.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// A reader for GIS models stored in plain text format. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on PlainTextGISModelReader.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class PlainTextGisModelReader : GisModelReader + { + private StreamReader mInput; + + /// + /// Constructor which directly instantiates the StreamReader containing + /// the model contents. + /// + /// + /// The StreamReader containing the model information. + /// + public PlainTextGisModelReader(StreamReader reader) + { + using (mInput = reader) + { + base.ReadModel(); + } + } + + /// + /// Constructor which takes a file and creates a reader for it. + /// + /// + /// The full path and file name in which the model is stored. + /// + public PlainTextGisModelReader(string fileName) + { + using (mInput = new StreamReader(fileName, System.Text.Encoding.UTF7)) + { + base.ReadModel(); + } + } + + /// + /// Reads a 32-bit signed integer from the model file. + /// + protected override int ReadInt32() + { + return int.Parse(mInput.ReadLine(), System.Globalization.CultureInfo.InvariantCulture); + } + + /// + /// Reads a double-precision floating point number from the model file. + /// + protected override double ReadDouble() + { + return double.Parse(mInput.ReadLine(), System.Globalization.CultureInfo.InvariantCulture); + } + + /// + /// Reads a string from the model file. + /// + protected override string ReadString() + { + return mInput.ReadLine(); + } + + } +} diff --git a/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelWriter.cs b/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelWriter.cs new file mode 100644 index 00000000..7dd5d433 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/IO/PlainTextGisModelWriter.cs @@ -0,0 +1,134 @@ +// Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the PlainTextGISModelReader.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; + +namespace BotSharp.MachineLearning.IO +{ + /// + /// Model writer that saves models in plain text format. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on PlainTextGISModelWriter.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class PlainTextGisModelWriter : GisModelWriter + { + private StreamWriter mOutput; + + /// + /// Default constructor. + /// + public PlainTextGisModelWriter() + { + } + + /// + /// Takes a GIS model and a file and writes the model to that file. + /// + /// + /// The GisModel which is to be persisted. + /// + /// + /// The name of the file in which the model is to be persisted. + /// + public void Persist(GisModel model, string fileName) + { + using (mOutput = new StreamWriter(fileName, false, System.Text.Encoding.UTF7)) + { + base.Persist(model); + } + } + + /// + /// Takes a GisModel and a stream and writes the model to that stream. + /// + /// + /// The GisModel which is to be persisted. + /// + /// + /// The StreamWriter which will be used to persist the model. + /// + public void Persist(GisModel model, StreamWriter writer) + { + using (mOutput = writer) + { + base.Persist(model); + } + } + + /// + /// Writes a string to the model file. + /// + /// /// + /// The string data to be persisted. + /// + protected override void WriteString(string data) + { + mOutput.Write(data); + mOutput.WriteLine(); + } + + /// + /// Writes a 32-bit signed integer to the model file. + /// + /// + /// The integer data to be persisted. + /// + protected override void WriteInt32(int data) + { + mOutput.Write(data.ToString(System.Globalization.CultureInfo.InvariantCulture)); + mOutput.WriteLine(); + } + + /// + /// Writes a double-precision floating point number to the model file. + /// + /// + /// The floating point data to be persisted. + /// + protected override void WriteDouble(double data) + { + mOutput.Write(data.ToString(System.Globalization.CultureInfo.InvariantCulture)); + mOutput.WriteLine(); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/ITrainingDataIndexer.cs b/BotSharp.MachineLearning/Entropy/ITrainingDataIndexer.cs new file mode 100644 index 00000000..458c4459 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/ITrainingDataIndexer.cs @@ -0,0 +1,86 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the DataIndexer.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +//Copyright (C) 2003 Thomas Morton +// +//This library is free software; you can redistribute it and/or +//modify it under the terms of the GNU Lesser General Public +//License as published by the Free Software Foundation; either +//version 2.1 of the License, or (at your option) any later version. +// +//This library is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU Lesser General Public +//License along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// Object that compresses events in memory and performs feature selection. + /// + public interface ITrainingDataIndexer + { + + /// + /// Gets an array of context data calculated from the training data. + /// + /// + /// Array of integer arrays, each containing the context data for an event. + /// + int[][] GetContexts(); + + /// + /// Gets an array indicating how many times each event is seen. + /// + /// + /// Integer array with event frequencies. + /// + int[] GetNumTimesEventsSeen(); + + /// + /// Gets an outcome list. + /// + /// + /// Integer array of outcomes. + /// + int[] GetOutcomeList(); + + /// + /// Gets an array of predicate labels. + /// + /// + /// Array of predicate labels. + /// + string[] GetPredicateLabels(); + + /// + /// Gets an array of outcome labels. + /// + /// + /// Array of outcome labels. + /// + string[] GetOutcomeLabels(); + } +} diff --git a/BotSharp.MachineLearning/Entropy/ITrainingDataReader.cs b/BotSharp.MachineLearning/Entropy/ITrainingDataReader.cs new file mode 100644 index 00000000..8ade4001 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/ITrainingDataReader.cs @@ -0,0 +1,74 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the DataStream.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// A interface for objects which can deliver a stream of training data to be + /// supplied to an ITrainingEventReader. It is not necessary to use a ITrainingDataReader in a + /// SharpEntropy application, but it can be used to support a wider variety of formats + /// in which your training data can be held. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on DataStream.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public interface ITrainingDataReader + { + /// + /// Returns the next slice of data held in this ITrainingDataReader. + /// + /// + /// the object representing the data which is next in this + /// ITrainingDataReader + /// + T NextToken(); + + /// + /// Test whether there are any training data items remaining in this ITrainingDataReader. + /// + /// + /// true if this ITrainingDataReader has more data tokens + /// + bool HasNext(); + } +} diff --git a/BotSharp.MachineLearning/Entropy/ITrainingEventReader.cs b/BotSharp.MachineLearning/Entropy/ITrainingEventReader.cs new file mode 100644 index 00000000..59b195db --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/ITrainingEventReader.cs @@ -0,0 +1,66 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the EventStream.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// An object which can deliver a stream of training events for the GIS + /// procedure (or others such as IIS if and when they are implemented). + /// TrainingEventReaders don't need to use SharpEntropy.ITrainingDataReader, but doing so + /// would provide greater flexibility for producing events from data stored in + /// different formats. + /// + public interface ITrainingEventReader + { + + /// + /// Returns the next TrainingEvent object held in this TrainingEventReader. + /// + /// + /// the TrainingEvent object which is next in this TrainingEventReader + /// + TrainingEvent ReadNextEvent(); + + /// + /// Test whether there are any TrainingEvents remaining in this TrainingEventReader. + /// + /// + /// true if this TrainingEventReader has more TrainingEvents + /// + bool HasNext(); + } +} diff --git a/BotSharp.MachineLearning/Entropy/OnePassDataIndexer.cs b/BotSharp.MachineLearning/Entropy/OnePassDataIndexer.cs new file mode 100644 index 00000000..10e26c44 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/OnePassDataIndexer.cs @@ -0,0 +1,212 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the OnePassDataIndexer.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections.Generic; + +namespace BotSharp.MachineLearning +{ + /// + /// An indexer for maxent model data which handles cutoffs for uncommon + /// contextual predicates and provides a unique integer index for each of the + /// predicates. The data structures built in the constructor of this class are + /// used by the GIS trainer. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on OnePassDataIndexer.java, $Revision: 1.1 $, $Date: 2003/12/13 16:41:29 $ + /// + public class OnePassDataIndexer : AbstractDataIndexer + { + /// + /// One argument constructor for OnePassDataIndexer which calls the two argument + /// constructor assuming no cutoff. + /// + /// + /// An ITrainingEventReader which contains the a list of all the Events + /// seen in the training data. + /// + public OnePassDataIndexer(ITrainingEventReader eventReader) : this(eventReader, 0) + { + } + + /// + /// Two argument constructor for OnePassDataIndexer. + /// + /// + /// An ITrainingEventReader which contains the a list of all the Events + /// seen in the training data. + /// + /// + /// The minimum number of times a predicate must have been + /// observed in order to be included in the model. + /// + public OnePassDataIndexer(ITrainingEventReader eventReader, int cutoff) + { + Dictionary predicateIndex; + List events; + List eventsToCompare; + + predicateIndex = new Dictionary(); + //NotifyProgress("Indexing events using cutoff of " + cutoff + "\n"); + + //NotifyProgress("\tComputing event counts... "); + events = ComputeEventCounts(eventReader, predicateIndex, cutoff); + //NotifyProgress("done. " + events.Count + " events"); + + //NotifyProgress("\tIndexing... "); + eventsToCompare = Index(events, predicateIndex); + + //NotifyProgress("done."); + + //NotifyProgress("Sorting and merging oEvents... "); + SortAndMerge(eventsToCompare); + //NotifyProgress("Done indexing."); + } + + /// + /// Reads events from eventReader into a List<TrainingEvent>. The + /// predicates associated with each event are counted and any which + /// occur at least cutoff times are added to the + /// predicatesInOut dictionary along with a unique integer index. + /// + /// + /// an ITrainingEventReader value + /// + /// + /// a Dictionary value + /// + /// + /// an int value + /// + /// + /// an List of TrainingEvents value + /// + private List ComputeEventCounts(ITrainingEventReader eventReader, Dictionary predicatesInOut, int cutoff) + { + var counter = new Dictionary(); + var events = new List(); + int predicateIndex = 0; + while (eventReader.HasNext()) + { + TrainingEvent trainingEvent = eventReader.ReadNextEvent(); + events.Add(trainingEvent); + string[] eventContext = trainingEvent.Context; + for (int currentEventContext = 0; currentEventContext < eventContext.Length; currentEventContext++) + { + if (!predicatesInOut.ContainsKey(eventContext[currentEventContext])) + { + if (counter.ContainsKey(eventContext[currentEventContext])) + { + counter[eventContext[currentEventContext]]++; + } + else + { + counter.Add(eventContext[currentEventContext], 1); + } + if (counter[eventContext[currentEventContext]] >= cutoff) + { + predicatesInOut.Add(eventContext[currentEventContext], predicateIndex++); + counter.Remove(eventContext[currentEventContext]); + } + } + } + } + return events; + } + + private List Index(List events, Dictionary predicateIndex) + { + var map = new Dictionary(); + + int eventCount = events.Count; + int outcomeCount = 0; + + var eventsToCompare = new List(eventCount); + var indexedContext = new List(); + + for (int eventIndex = 0; eventIndex < eventCount; eventIndex++) + { + TrainingEvent currentTrainingEvent = events[eventIndex]; + string[] eventContext = currentTrainingEvent.Context; + ComparableEvent comparableEvent; + + int outcomeIndex; + + string outcome = currentTrainingEvent.Outcome; + + if (map.ContainsKey(outcome)) + { + outcomeIndex = map[outcome]; + } + else + { + outcomeIndex = outcomeCount++; + map.Add(outcome, outcomeIndex); + } + + for (int currentEventContext = 0; currentEventContext < eventContext.Length; currentEventContext++) + { + string predicate = eventContext[currentEventContext]; + if (predicateIndex.ContainsKey(predicate)) + { + indexedContext.Add(predicateIndex[predicate]); + } + } + + // drop events with no active features + if (indexedContext.Count > 0) + { + comparableEvent = new ComparableEvent(outcomeIndex, indexedContext.ToArray()); + eventsToCompare.Add(comparableEvent); + } + else + { + //"Dropped event " + oEvent.Outcome + ":" + oEvent.Context); + } + // recycle the list + indexedContext.Clear(); + } + SetOutcomeLabels(ToIndexedStringArray(map)); + SetPredicateLabels(ToIndexedStringArray(predicateIndex)); + return eventsToCompare; + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/PatternedPredicate.cs b/BotSharp.MachineLearning/Entropy/PatternedPredicate.cs new file mode 100644 index 00000000..b09e5680 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/PatternedPredicate.cs @@ -0,0 +1,118 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// Object containing predicate data, where the parameters are matched to + /// the outcomes in an outcome pattern. + /// + /// + /// Richard J. Northedge + /// + public class PatternedPredicate + { + private int mOutcomePattern; + private double[] mParameters; + private string mName; + + /// + /// Creates a PatternedPredicate object. + /// + /// + /// Index into the outcome pattern array, specifying which outcome pattern relates to + /// this predicate. + /// + /// + /// Array of parameters for this predicate. + /// + protected internal PatternedPredicate(int outcomePattern, double[] parameters) + { + mOutcomePattern = outcomePattern; + mParameters = parameters; + } + + /// + /// Creates a PatternedPredicate object. + /// + /// + /// The predicate name. + /// + /// + /// Array of parameters for this predicate. + /// + protected internal PatternedPredicate(string name, double[] parameters) + { + mName = name; + mParameters = parameters; + } + + /// + /// Index into array of outcome patterns. + /// + public int OutcomePattern + { + get + { + return mOutcomePattern; + } + set // for trainer + { + mOutcomePattern = value; + } + } + + /// + /// Gets the value of a parameter from this predicate. + /// + /// + /// index into the parameter array. + /// + /// + public double GetParameter(int index) + { + return mParameters[index]; + } + + /// + /// Number of parameters associated with this predicate. + /// + public int ParameterCount + { + get + { + return mParameters.Length; + } + } + + /// + /// Name of the predicate. + /// + public string Name + { + get + { + return mName; + } + set + { + mName = value; + } + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/PlainTextByLineDataReader.cs b/BotSharp.MachineLearning/Entropy/PlainTextByLineDataReader.cs new file mode 100644 index 00000000..ece21860 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/PlainTextByLineDataReader.cs @@ -0,0 +1,86 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the PlainTextByLineDataStream.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.IO; + +namespace BotSharp.MachineLearning +{ + /// + /// This ITrainingDataReader implementation will take care of reading a plain text file + /// and returning the strings between each new line character, which is what + /// many SharpEntropy applications need in order to create ITrainingEventReaders. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on PlainTextByLineDataStream.java, $Revision: 1.1.1.1 $, $Date: 2001/10/23 14:06:53 $ + /// + public class PlainTextByLineDataReader : ITrainingDataReader + { + private readonly StreamReader _dataReader; + private string _nextLine; + + /// + /// Creates a training data reader for reading text lines from a file or other text stream + /// + /// StreamReader containing the source of the training data + public PlainTextByLineDataReader(StreamReader dataSource) + { + _dataReader = dataSource; + _nextLine = _dataReader.ReadLine(); + } + + /// Gets the next text line from the training data + /// Next text line from the training data + public virtual string NextToken() + { + string currentLine = _nextLine; + _nextLine = _dataReader.ReadLine(); + return currentLine; + } + + /// Checks if there is any more training data + /// true if there is more training data to be read + public virtual bool HasNext() + { + return (_nextLine != null); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/TrainingEvent.cs b/BotSharp.MachineLearning/Entropy/TrainingEvent.cs new file mode 100644 index 00000000..3fde7f6b --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/TrainingEvent.cs @@ -0,0 +1,94 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the Event.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +// Copyright (C) 2001 Jason Baldridge and Gann Bierner +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; + +namespace BotSharp.MachineLearning +{ + /// + /// The context of a decision point during training. This includes + /// contextual predicates and an outcome. + /// + /// + /// Jason Baldridge + /// + /// + /// Richard J. Northedge + /// + /// + /// based on Event.java, $Revision: 1.3 $, $Date: 2003/12/09 23:13:08 $ + /// + public class TrainingEvent + { + /// + /// The outcome label for this training event. + /// + public string Outcome { get; private set; } + + /// + /// The context for this training event. + /// + /// + /// A string array of context values for this training event. + /// + public string[] Context { get; private set; } + + /// + /// Constructor for a training event. + /// + /// + /// the outcome label + /// + /// + /// array containing context values + /// + public TrainingEvent(string outcome, string[] context) + { + Outcome = outcome; + Context = context; + } + + /// + /// Override providing text summary of the training event. + /// + /// + /// Summary of the training event. + /// + public override string ToString() + { + return Outcome + " " + string.Join(", ", Context); + } + } +} diff --git a/BotSharp.MachineLearning/Entropy/TwoPassDataIndexer.cs b/BotSharp.MachineLearning/Entropy/TwoPassDataIndexer.cs new file mode 100644 index 00000000..f5f37ac2 --- /dev/null +++ b/BotSharp.MachineLearning/Entropy/TwoPassDataIndexer.cs @@ -0,0 +1,282 @@ +//Copyright (C) 2005 Richard J. Northedge +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +//This file is based on the TwoPassDataIndexer.java source file found in the +//original java implementation of MaxEnt. That source file contains the following header: + +//Copyright (C) 2003 Thomas Morton +// +//This library is free software; you can redistribute it and/or +//modify it under the terms of the GNU Lesser General Public +//License as published by the Free Software Foundation; either +//version 2.1 of the License, or (at your option) any later version. +// +//This library is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU Lesser General Public +//License along with this program; if not, write to the Free Software +//Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace BotSharp.MachineLearning +{ + /// + /// Collecting event and context counts by making two passes over the events. + /// The first pass determines which contexts will be used by the model, and the second + /// pass creates the events in memory containing only the contexts which will be used. + /// This greatly reduces the amount of memory required for storing the events. + /// During the first pass a temporary event file is created which is read during the second pass. + /// + /// /// + /// Tom Morton + /// + /// /// /// + /// Richard J. Northedge + /// + public class TwoPassDataIndexer : AbstractDataIndexer + { + /// + /// One argument constructor for DataIndexer which calls the two argument + /// constructor assuming no cutoff. + /// + /// + /// An ITrainingEventReader which contains the list of all the events + /// seen in the training data. + /// + public TwoPassDataIndexer(ITrainingEventReader eventReader): this(eventReader, 0){} + + /// + /// Two argument constructor for TwoPassDataIndexer. + /// + /// + /// An ITrainingEventReader which contains the a list of all the events + /// seen in the training data. + /// + /// + /// The minimum number of times a predicate must have been + /// observed in order to be included in the model. + /// + public TwoPassDataIndexer(ITrainingEventReader eventReader, int cutoff) + { + List eventsToCompare; + + var predicateIndex = new Dictionary(); + //NotifyProgress("Indexing events using cutoff of " + cutoff + "\n"); + + //NotifyProgress("\tComputing event counts... "); + + string tempFile = new FileInfo(Path.GetTempFileName()).FullName; + + int eventCount = ComputeEventCounts(eventReader, tempFile, predicateIndex, cutoff); + //NotifyProgress("done. " + eventCount + " events"); + + //NotifyProgress("\tIndexing... "); + + using (var fileEventReader = new FileEventReader(tempFile)) + { + eventsToCompare = Index(eventCount, fileEventReader, predicateIndex); + } + + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + + //NotifyProgress("done."); + + //NotifyProgress("Sorting and merging events... "); + SortAndMerge(eventsToCompare); + //NotifyProgress("Done indexing."); + } + + /// + /// Reads events from eventStream into a dictionary. The + /// predicates associated with each event are counted and any which + /// occur at least cutoff times are added to the + /// predicatesInOut map along with a unique integer index. + /// + /// + /// an ITrainingEventReader value + /// + /// + /// a file name to which the events are written to for later processing. + /// + /// + /// a Dictionary value + /// + /// + /// an int value + /// + private int ComputeEventCounts(ITrainingEventReader eventReader, string eventStoreFile, Dictionary predicatesInOut, int cutoff) + { + var counter = new Dictionary(); + int predicateIndex = 0; + int eventCount = 0; + + using (var eventStoreWriter = new StreamWriter(eventStoreFile)) + { + while (eventReader.HasNext()) + { + TrainingEvent currentTrainingEvent = eventReader.ReadNextEvent(); + eventCount++; + eventStoreWriter.Write(FileEventReader.ToLine(currentTrainingEvent)); + string[] eventContext = currentTrainingEvent.Context; + for (int currentPredicate = 0; currentPredicate < eventContext.Length; currentPredicate++) + { + if (!predicatesInOut.ContainsKey(eventContext[currentPredicate])) + { + if (counter.ContainsKey(eventContext[currentPredicate])) + { + counter[eventContext[currentPredicate]]++; + } + else + { + counter.Add(eventContext[currentPredicate], 1); + } + if (counter[eventContext[currentPredicate]] >= cutoff) + { + predicatesInOut.Add(eventContext[currentPredicate], predicateIndex++); + counter.Remove(eventContext[currentPredicate]); + } + } + } + } + } + return eventCount; + } + + private List Index(int eventCount, ITrainingEventReader eventReader, Dictionary predicateIndex) + { + var outcomeMap = new Dictionary(); + int outcomeCount = 0; + var eventsToCompare = new List(eventCount); + var indexedContext = new List(); + while (eventReader.HasNext()) + { + TrainingEvent currentTrainingEvent = eventReader.ReadNextEvent(); + string[] eventContext = currentTrainingEvent.Context; + ComparableEvent comparableEvent; + + int outcomeId; + string outcome = currentTrainingEvent.Outcome; + + if (outcomeMap.ContainsKey(outcome)) + { + outcomeId = outcomeMap[outcome]; + } + else + { + outcomeId = outcomeCount++; + outcomeMap.Add(outcome, outcomeId); + } + + for (int currentPredicate = 0; currentPredicate < eventContext.Length; currentPredicate++) + { + string predicate = eventContext[currentPredicate]; + if (predicateIndex.ContainsKey(predicate)) + { + indexedContext.Add(predicateIndex[predicate]); + } + } + + // drop events with no active features + if (indexedContext.Count > 0) + { + comparableEvent = new ComparableEvent(outcomeId, indexedContext.ToArray()); + eventsToCompare.Add(comparableEvent); + } + else + { + //"Dropped event " + currentTrainingEvent.Outcome + ":" + currentTrainingEvent.Context); + } + // recycle the list + indexedContext.Clear(); + } + SetOutcomeLabels(ToIndexedStringArray(outcomeMap)); + SetPredicateLabels(ToIndexedStringArray(predicateIndex)); + return eventsToCompare; + } + } + + class FileEventReader : ITrainingEventReader, IDisposable + { + private StreamReader mReader; + private string mCurrentLine; + + private char[] mWhitespace; + + public FileEventReader(string fileName) + { + mReader = new StreamReader(fileName, Encoding.UTF7); + mWhitespace = new char[] {'\t', '\n', '\r', ' '}; + } + + public virtual bool HasNext() + { + mCurrentLine = mReader.ReadLine(); + return (mCurrentLine != null); + } + + public virtual TrainingEvent ReadNextEvent() + { + string[] tokens = mCurrentLine.Split(mWhitespace); + string outcome = tokens[0]; + var context = new string[tokens.Length - 1]; + Array.Copy(tokens, 1, context, 0, tokens.Length - 1); + + return (new TrainingEvent(outcome, context)); + } + + public static string ToLine(TrainingEvent eventToConvert) + { + var lineBuilder = new StringBuilder(); + lineBuilder.Append(eventToConvert.Outcome); + string[] context = eventToConvert.Context; + for (int contextIndex = 0, contextLength = context.Length; contextIndex < contextLength; contextIndex++) + { + lineBuilder.Append(" " + context[contextIndex]); + } + lineBuilder.Append(System.Environment.NewLine); + return lineBuilder.ToString(); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + mReader.Close(); + } + } + + ~FileEventReader() + { + Dispose (false); + } + } +} diff --git a/BotSharp.MachineLearning/WordNet/DataFileEngine.cs b/BotSharp.MachineLearning/WordNet/DataFileEngine.cs index 3a4937b5..b835f4b1 100644 --- a/BotSharp.MachineLearning/WordNet/DataFileEngine.cs +++ b/BotSharp.MachineLearning/WordNet/DataFileEngine.cs @@ -18,7 +18,7 @@ using System; using System.IO; using System.Collections.Generic; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for DataFileEngine. diff --git a/BotSharp.MachineLearning/WordNet/IndexWord.cs b/BotSharp.MachineLearning/WordNet/IndexWord.cs index cee84981..10b091eb 100644 --- a/BotSharp.MachineLearning/WordNet/IndexWord.cs +++ b/BotSharp.MachineLearning/WordNet/IndexWord.cs @@ -17,7 +17,7 @@ using System; using System.Linq; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for IndexWord. diff --git a/BotSharp.MachineLearning/WordNet/Morph/AbstractDelegatingOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/AbstractDelegatingOperation.cs index 854e64db..a8c172b1 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/AbstractDelegatingOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/AbstractDelegatingOperation.cs @@ -21,7 +21,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { public abstract class AbstractDelegatingOperation : IOperation { diff --git a/BotSharp.MachineLearning/WordNet/Morph/DetachSuffixesOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/DetachSuffixesOperation.cs index ffe69c0c..81f5395c 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/DetachSuffixesOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/DetachSuffixesOperation.cs @@ -21,7 +21,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { /// /// Remove all applicable suffixes from the word(s) and do a look-up. diff --git a/BotSharp.MachineLearning/WordNet/Morph/IOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/IOperation.cs index 11b9bf25..4b09e4b3 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/IOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/IOperation.cs @@ -21,7 +21,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { public interface IOperation { diff --git a/BotSharp.MachineLearning/WordNet/Morph/LookupExceptionsOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/LookupExceptionsOperation.cs index 0cf77509..82c7fc85 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/LookupExceptionsOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/LookupExceptionsOperation.cs @@ -21,7 +21,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { /// Lookup the word in the exceptions file of the given part-of-speech. public class LookupExceptionsOperation : IOperation diff --git a/BotSharp.MachineLearning/WordNet/Morph/LookupIndexWordOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/LookupIndexWordOperation.cs index 8563e276..9b923348 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/LookupIndexWordOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/LookupIndexWordOperation.cs @@ -21,7 +21,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { public class LookupIndexWordOperation : IOperation { diff --git a/BotSharp.MachineLearning/WordNet/Morph/TokenizerOperation.cs b/BotSharp.MachineLearning/WordNet/Morph/TokenizerOperation.cs index b8a15eb1..c11314e5 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/TokenizerOperation.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/TokenizerOperation.cs @@ -22,7 +22,7 @@ using System.Collections.Generic; using System.Text; using System.Collections; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { public class TokenizerOperation : AbstractDelegatingOperation { diff --git a/BotSharp.MachineLearning/WordNet/Morph/Util.cs b/BotSharp.MachineLearning/WordNet/Morph/Util.cs index 6f4ff512..bcabf79f 100644 --- a/BotSharp.MachineLearning/WordNet/Morph/Util.cs +++ b/BotSharp.MachineLearning/WordNet/Morph/Util.cs @@ -22,7 +22,7 @@ using System.Collections; using System.Collections.Generic; using System.Text; -namespace SharpWordNet.Morph +namespace BotSharp.MachineLearning.Morph { public class Util { diff --git a/BotSharp.MachineLearning/WordNet/Relation.cs b/BotSharp.MachineLearning/WordNet/Relation.cs index d4a50e6d..6db3c45b 100644 --- a/BotSharp.MachineLearning/WordNet/Relation.cs +++ b/BotSharp.MachineLearning/WordNet/Relation.cs @@ -16,7 +16,7 @@ using System; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for Relation. diff --git a/BotSharp.MachineLearning/WordNet/RelationType.cs b/BotSharp.MachineLearning/WordNet/RelationType.cs index 20b12e55..98e72e7a 100644 --- a/BotSharp.MachineLearning/WordNet/RelationType.cs +++ b/BotSharp.MachineLearning/WordNet/RelationType.cs @@ -16,7 +16,7 @@ using System; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for RelationType. diff --git a/BotSharp.MachineLearning/WordNet/Synset.cs b/BotSharp.MachineLearning/WordNet/Synset.cs index daf6eb55..1cc5e927 100644 --- a/BotSharp.MachineLearning/WordNet/Synset.cs +++ b/BotSharp.MachineLearning/WordNet/Synset.cs @@ -16,7 +16,7 @@ using System; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for Synset. diff --git a/BotSharp.MachineLearning/WordNet/Tokenizer.cs b/BotSharp.MachineLearning/WordNet/Tokenizer.cs index 5f538ffd..bbe32c6b 100644 --- a/BotSharp.MachineLearning/WordNet/Tokenizer.cs +++ b/BotSharp.MachineLearning/WordNet/Tokenizer.cs @@ -16,7 +16,7 @@ using System; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for Tokenizer. diff --git a/BotSharp.MachineLearning/WordNet/WordNetEngine.cs b/BotSharp.MachineLearning/WordNet/WordNetEngine.cs index 1b437523..2f0dbecf 100644 --- a/BotSharp.MachineLearning/WordNet/WordNetEngine.cs +++ b/BotSharp.MachineLearning/WordNet/WordNetEngine.cs @@ -17,7 +17,7 @@ using System; using System.Collections.Generic; -namespace SharpWordNet +namespace BotSharp.MachineLearning { /// /// Summary description for WordNetEngine.