SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add WordNet algorithm

Browse source
This commit is contained in:
Oceania2018 2018-07-28 12:56:15 -05:00
parent 880488ea18
commit 535854df1f
16 changed files with 1602 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Core/BotSharp.Core.csproj
View file

@ -36,7 +36,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="DotNetToolkit" Version="1.4.0" /> <PackageReference Include="DotNetToolkit" Version="1.4.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.6.3" /> <PackageReference Include="EntityFrameworkCore.BootKit" Version="1.7.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" /> <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="RestSharp" Version="106.3.1" /> <PackageReference Include="RestSharp" Version="106.3.1" />
</ItemGroup> </ItemGroup>

32
BotSharp.Core/Engines/Dialogflow/ApiAi.cs
View file

@ -10,21 +10,6 @@ namespace BotSharp.Core.Engines.Dialogflow
{ {
private AIDataService dataService; private AIDataService dataService;
public AIResponse TextRequest(string text)
{
if (dataService == null)
{
dataService = new AIDataService(AiConfig);
}
if (string.IsNullOrEmpty(text))
{
throw new ArgumentNullException("text");
}
return TextRequest(new AIRequest(text));
}
public AIResponse TextRequest(AIRequest request) public AIResponse TextRequest(AIRequest request)
{ {
if (request == null) if (request == null)
@ -40,24 +25,9 @@ namespace BotSharp.Core.Engines.Dialogflow
return dataService.Request(request); return dataService.Request(request);
} }
public AIResponse TextRequest(string text, RequestExtras requestExtras)
{
if (string.IsNullOrEmpty(text))
{
throw new ArgumentNullException("text");
}
if (dataService == null)
{
dataService = new AIDataService(AiConfig);
}
return TextRequest(new AIRequest(text, requestExtras));
}
public void Train() public void Train()
{ {
throw new NotImplementedException();
} }
} }
} }

511
BotSharp.MachineLearning/WordNet/DataFileEngine.cs Normal file
View file

@ -0,0 +1,511 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
using System.IO;
using System.Collections.Generic;
namespace SharpWordNet
{
/// <summary>
/// Summary description for DataFileEngine.
/// </summary>
public class DataFileEngine : WordNetEngine
{
private readonly string _dataFolder;
private readonly Dictionary<string, PosDataFileSet> _dataFileDictionary;
private string[] _lexicographerFiles;
private Dictionary<string, RelationType> _relationTypeDictionary;
// Public Methods (class specific) ------------------
public string DataFolder
{
get
{
return _dataFolder;
}
}
public DataFileEngine(string dataFolder)
{
_dataFolder = dataFolder;
_dataFileDictionary = new Dictionary<string, PosDataFileSet>(4)
{
{"noun", new PosDataFileSet(dataFolder, "noun")},
{"verb", new PosDataFileSet(dataFolder, "verb")},
{"adjective", new PosDataFileSet(dataFolder, "adj")},
{"adverb", new PosDataFileSet(dataFolder, "adv")}
};
InitializeLexicographerFiles();
InitializeRelationTypes();
}
// abstract methods implementation ------------------
public override string[] GetPartsOfSpeech()
{
return new List<string>(_dataFileDictionary.Keys).ToArray();
}
public override string[] GetPartsOfSpeech(string lemma)
{
var partsOfSpeech = new List<string>();
foreach (string partOfSpeech in _dataFileDictionary.Keys)
{
if (BinarySearch(lemma, _dataFileDictionary[partOfSpeech].IndexFile) != null)
{
partsOfSpeech.Add(partOfSpeech);
}
}
return partsOfSpeech.ToArray();
}
public override IndexWord[] GetAllIndexWords(string partOfSpeech)
{
StreamReader searchFile = _dataFileDictionary[partOfSpeech].IndexFile;
string line;
string space = " ";
var indexWords = new List<IndexWord>();
searchFile.DiscardBufferedData();
searchFile.BaseStream.Position = 0;
while (!searchFile.EndOfStream)
{
line = searchFile.ReadLine();
if (!line.StartsWith(space))
{
indexWords.Add(CreateIndexWord(partOfSpeech, line));
}
}
return indexWords.ToArray();
}
public override IndexWord GetIndexWord(string lemma, string partOfSpeech)
{
string line = BinarySearch(lemma, _dataFileDictionary[partOfSpeech].IndexFile);
if (line != null)
{
return CreateIndexWord(partOfSpeech, line);
}
return null;
}
public override Synset[] GetSynsets(string lemma)
{
var synsets = new List<Synset>();
foreach (string partOfSpeech in _dataFileDictionary.Keys)
{
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
if (indexWord != null)
{
foreach (int synsetOffset in indexWord.SynsetOffsets)
{
Synset synset = CreateSynset(partOfSpeech, synsetOffset);
synsets.Add(synset);
}
}
}
return synsets.ToArray();
}
public override Synset[] GetSynsets(string lemma, string partOfSpeech)
{
var synsets = new List<Synset>();
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
if (indexWord != null)
{
foreach (int synsetOffset in indexWord.SynsetOffsets)
{
Synset synset = CreateSynset(partOfSpeech, synsetOffset);
synsets.Add(synset);
}
}
return synsets.ToArray();
}
public override RelationType[] GetRelationTypes(string lemma, string partOfSpeech)
{
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
if (indexWord != null)
{
if (indexWord.RelationTypes != null)
{
int relationTypeCount = indexWord.RelationTypes.Length;
var relationTypes = new RelationType[relationTypeCount];
for (int currentRelationType = 0; currentRelationType < relationTypeCount; currentRelationType++)
{
relationTypes[currentRelationType] = _relationTypeDictionary[indexWord.RelationTypes[currentRelationType]];
}
return relationTypes;
}
return null;
}
return null;
}
public override Synset GetSynset(string lemma, string partOfSpeech, int senseNumber)
{
if (senseNumber < 1)
{
throw new ArgumentOutOfRangeException("senseNumber", senseNumber, "cannot be less than 1");
}
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
if (indexWord != null)
{
if (senseNumber > (indexWord.SynsetOffsets.Length + 1))
{
return (null);
}
Synset synset = CreateSynset(partOfSpeech, indexWord.SynsetOffsets[senseNumber - 1]);
return (synset);
}
return null;
}
// Private Methods----------------------------------
private string BinarySearch(string searchKey, StreamReader searchFile)
{
if (searchKey.Length == 0)
{
return null;
}
int c,n;
long top,bot,mid,diff;
string line,key;
diff = 666;
line = "";
bot = searchFile.BaseStream.Seek(0, SeekOrigin.End);
top = 0;
mid = (bot-top)/2;
do
{
searchFile.DiscardBufferedData();
searchFile.BaseStream.Position = mid - 1;
if (mid != 1)
{
while ((c = searchFile.Read()) != '\n' && c != -1) { }
}
line = searchFile.ReadLine();
if (line == null)
{
return null;
}
n = line.IndexOf(' ');
key = line.Substring(0,n);
key=key.Replace("-"," ").Replace("_"," ");
if (string.CompareOrdinal(key, searchKey) < 0)
{
top = mid;
diff = (bot - top)/2;
mid = top + diff;
}
if (string.CompareOrdinal(key, searchKey) > 0)
{
bot = mid;
diff = (bot - top)/2;
mid = top + diff;
}
} while (key!=searchKey && diff!=0);
if (key == searchKey)
{
return line;
}
return null;
}
private IndexWord CreateIndexWord(string partOfSpeech, string line)
{
var tokenizer = new Tokenizer(line);
string word = tokenizer.NextToken().Replace('_', ' ');
string redundantPartOfSpeech = tokenizer.NextToken();
int senseCount = int.Parse(tokenizer.NextToken());
int relationTypeCount = int.Parse(tokenizer.NextToken());
string[] relationTypes = null;
if (relationTypeCount > 0)
{
relationTypes = new string[relationTypeCount];
for (int currentRelationType = 0; currentRelationType < relationTypeCount; currentRelationType++)
{
relationTypes[currentRelationType] = tokenizer.NextToken();
}
}
int redundantSenseCount = int.Parse(tokenizer.NextToken());
int tagSenseCount = int.Parse(tokenizer.NextToken());
int[] synsetOffsets = null;
if (senseCount > 0)
{
synsetOffsets = new int[senseCount];
for (int currentOffset = 0; currentOffset < senseCount; currentOffset++)
{
synsetOffsets[currentOffset] = int.Parse(tokenizer.NextToken());
}
}
return new IndexWord(word, partOfSpeech, relationTypes, synsetOffsets, tagSenseCount);
}
protected internal override Synset CreateSynset(string partOfSpeech, int synsetOffset)
{
StreamReader dataFile = _dataFileDictionary[partOfSpeech].DataFile;
dataFile.DiscardBufferedData();
dataFile.BaseStream.Seek(synsetOffset, SeekOrigin.Begin);
string record = dataFile.ReadLine();
var tokenizer = new Tokenizer(record);
var nextToken = tokenizer.NextToken();
int offset = int.Parse(nextToken);
var nt = int.Parse(tokenizer.NextToken());
string lexicographerFile = _lexicographerFiles[nt];
string synsetType = tokenizer.NextToken();
int wordCount = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
var words = new string[wordCount];
for (int iCurrentWord = 0; iCurrentWord < wordCount; iCurrentWord++)
{
words[iCurrentWord] = tokenizer.NextToken().Replace("_", " ");
int uniqueID = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
}
int relationCount = int.Parse(tokenizer.NextToken());
var relations = new Relation[relationCount];
for (int currentRelation = 0; currentRelation < relationCount; currentRelation++)
{
string relationTypeKey = tokenizer.NextToken();
// if (fpos.name=="adj" && sstype==AdjSynSetType.DontKnow)
// {
// if (ptrs[j].ptp.mnemonic=="ANTPTR")
// sstype = AdjSynSetType.DirectAnt;
// else if (ptrs[j].ptp.mnemonic=="PERTPTR")
// sstype = AdjSynSetType.Pertainym;
// }
int targetSynsetOffset = int.Parse(tokenizer.NextToken());
string targetPartOfSpeech = tokenizer.NextToken();
switch (targetPartOfSpeech)
{
case "n":
targetPartOfSpeech = "noun";
break;
case "v":
targetPartOfSpeech = "verb";
break;
case "a":
case "s":
targetPartOfSpeech = "adjective";
break;
case "r":
targetPartOfSpeech = "adverb";
break;
}
int sourceTarget = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
if (sourceTarget == 0)
{
relations[currentRelation] = new Relation(this, (RelationType)_relationTypeDictionary[relationTypeKey], targetSynsetOffset, targetPartOfSpeech);
}
else
{
int sourceWord = sourceTarget >> 8;
int targetWord = sourceTarget & 0xff;
relations[currentRelation] = new Relation(this, (RelationType)_relationTypeDictionary[relationTypeKey], targetSynsetOffset, targetPartOfSpeech, sourceWord, targetWord);
}
}
string frameData = tokenizer.NextToken();
if (frameData != "|")
{
int frameCount = int.Parse(frameData);
for (int currentFrame = 0; currentFrame < frameCount; currentFrame++)
{
frameData = tokenizer.NextToken(); // +
int frameNumber = int.Parse(tokenizer.NextToken());
int wordID = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
}
frameData = tokenizer.NextToken();
}
string gloss = record.Substring(record.IndexOf('|') + 1);
var synset = new Synset(synsetOffset, gloss, words, lexicographerFile, relations);
return synset;
}
protected internal override string[] GetExceptionForms(string lemma, string partOfSpeech)
{
string line = BinarySearch(lemma, _dataFileDictionary[partOfSpeech].ExceptionFile);
if (line != null)
{
var exceptionForms = new List<string>();
var tokenizer = new Tokenizer(line);
string skipWord = tokenizer.NextToken();
string word = tokenizer.NextToken();
while (word != null)
{
exceptionForms.Add(word);
word = tokenizer.NextToken();
}
return exceptionForms.ToArray();
}
return mEmpty;
}
private void InitializeLexicographerFiles()
{
_lexicographerFiles = new string[45];
_lexicographerFiles[0] = "adj.all - all adjective clusters";
_lexicographerFiles[1] = "adj.pert - relational adjectives (pertainyms)";
_lexicographerFiles[2] = "adv.all - all adverbs";
_lexicographerFiles[3] = "noun.Tops - unique beginners for nouns";
_lexicographerFiles[4] = "noun.act - nouns denoting acts or actions";
_lexicographerFiles[5] = "noun.animal - nouns denoting animals";
_lexicographerFiles[6] = "noun.artifact - nouns denoting man-made objects";
_lexicographerFiles[7] = "noun.attribute - nouns denoting attributes of people and objects";
_lexicographerFiles[8] = "noun.body - nouns denoting body parts";
_lexicographerFiles[9] = "noun.cognition - nouns denoting cognitive processes and contents";
_lexicographerFiles[10] = "noun.communication - nouns denoting communicative processes and contents";
_lexicographerFiles[11] = "noun.event - nouns denoting natural events";
_lexicographerFiles[12] = "noun.feeling - nouns denoting feelings and emotions";
_lexicographerFiles[13] = "noun.food - nouns denoting foods and drinks";
_lexicographerFiles[14] = "noun.group - nouns denoting groupings of people or objects";
_lexicographerFiles[15] = "noun.location - nouns denoting spatial position";
_lexicographerFiles[16] = "noun.motive - nouns denoting goals";
_lexicographerFiles[17] = "noun.object - nouns denoting natural objects (not man-made)";
_lexicographerFiles[18] = "noun.person - nouns denoting people";
_lexicographerFiles[19] = "noun.phenomenon - nouns denoting natural phenomena";
_lexicographerFiles[20] = "noun.plant - nouns denoting plants";
_lexicographerFiles[21] = "noun.possession - nouns denoting possession and transfer of possession";
_lexicographerFiles[22] = "noun.process - nouns denoting natural processes";
_lexicographerFiles[23] = "noun.quantity - nouns denoting quantities and units of measure";
_lexicographerFiles[24] = "noun.relation - nouns denoting relations between people or things or ideas";
_lexicographerFiles[25] = "noun.shape - nouns denoting two and three dimensional shapes";
_lexicographerFiles[26] = "noun.state - nouns denoting stable states of affairs";
_lexicographerFiles[27] = "noun.substance - nouns denoting substances";
_lexicographerFiles[28] = "noun.time - nouns denoting time and temporal relations";
_lexicographerFiles[29] = "verb.body - verbs of grooming, dressing and bodily care";
_lexicographerFiles[30] = "verb.change - verbs of size, temperature change, intensifying, etc.";
_lexicographerFiles[31] = "verb.cognition - verbs of thinking, judging, analyzing, doubting";
_lexicographerFiles[32] = "verb.communication - verbs of telling, asking, ordering, singing";
_lexicographerFiles[33] = "verb.competition - verbs of fighting, athletic activities";
_lexicographerFiles[34] = "verb.consumption - verbs of eating and drinking";
_lexicographerFiles[35] = "verb.contact - verbs of touching, hitting, tying, digging";
_lexicographerFiles[36] = "verb.creation - verbs of sewing, baking, painting, performing";
_lexicographerFiles[37] = "verb.emotion - verbs of feeling";
_lexicographerFiles[38] = "verb.motion - verbs of walking, flying, swimming";
_lexicographerFiles[39] = "verb.perception - verbs of seeing, hearing, feeling";
_lexicographerFiles[40] = "verb.possession - verbs of buying, selling, owning";
_lexicographerFiles[41] = "verb.social - verbs of political and social activities and events";
_lexicographerFiles[42] = "verb.stative - verbs of being, having, spatial relations";
_lexicographerFiles[43] = "verb.weather - verbs of raining, snowing, thawing, thundering";
_lexicographerFiles[44] = "adj.ppl - participial adjectives";
}
private void InitializeRelationTypes()
{
_relationTypeDictionary = new Dictionary<string, RelationType>(30)
{
{"!", new RelationType("Antonym", new string[] {"noun", "verb", "adjective", "adverb"})},
{"@", new RelationType("Hypernym", new string[] {"noun", "verb"})},
{"@i", new RelationType("Instance Hypernym", new string[] {"noun"})},
{"~", new RelationType("Hyponym", new string[] {"noun", "verb"})},
{"~i", new RelationType("Instance Hyponym", new string[] {"noun"})},
{"#m", new RelationType("Member holonym", new string[] {"noun"})},
{"#s", new RelationType("Substance holonym", new string[] {"noun"})},
{"#p", new RelationType("Part holonym", new string[] {"noun"})},
{"%m", new RelationType("Member meronym", new string[] {"noun"})},
{"%s", new RelationType("Substance meronym", new string[] {"noun"})},
{"%p", new RelationType("Part meronym", new string[] {"noun"})},
{"=", new RelationType("Attribute", new string[] {"noun", "adjective"})},
{"+", new RelationType("Derivationally related form", new string[] {"noun", "verb"})},
{";c", new RelationType("Domain of synset - TOPIC", new string[] {"noun", "verb", "adjective", "adverb"})},
{"-c", new RelationType("Member of this domain - TOPIC", new string[] {"noun"})},
{";r", new RelationType("Domain of synset - REGION", new string[] {"noun", "verb", "adjective", "adverb"})},
{"-r", new RelationType("Member of this domain - REGION", new string[] {"noun"})},
{";u", new RelationType("Domain of synset - USAGE", new string[] {"noun", "verb", "adjective", "adverb"})},
{"-u", new RelationType("Member of this domain - USAGE", new string[] {"noun"})},
{"*", new RelationType("Entailment", new string[] {"verb"})},
{">", new RelationType("Cause", new string[] {"verb"})},
{"^", new RelationType("Also see", new string[] {"verb", "adjective"})},
{"$", new RelationType("Verb Group", new string[] {"verb"})},
{"&", new RelationType("Similar to", new string[] {"adjective"})},
{"<", new RelationType("Participle of verb", new string[] {"adjective"})},
{@"\", new RelationType("Pertainym", new string[] {"adjective", "adverb"})}
};
//moRelationTypeDictionary.Add(";", new RelationType("Domain of synset", new string[] {"noun", "verb", "adjective", "adverb"}));
//moRelationTypeDictionary.Add("-", new RelationType("Member of this domain", new string[] {"noun"}));
}
private class PosDataFileSet
{
private readonly StreamReader _indexFile;
private readonly StreamReader _dataFile;
private readonly StreamReader _exceptionFile;
public StreamReader IndexFile
{
get
{
return _indexFile;
}
}
public StreamReader DataFile
{
get
{
return _dataFile;
}
}
public StreamReader ExceptionFile
{
get
{
return _exceptionFile;
}
}
public PosDataFileSet(string dataFolder, string partOfSpeech)
{
_indexFile = new StreamReader(Path.Combine(dataFolder, "index." + partOfSpeech));
_dataFile = new StreamReader(Path.Combine(dataFolder, "data." + partOfSpeech));
_exceptionFile = new StreamReader(Path.Combine(dataFolder, partOfSpeech + ".exc"));
}
}
}
}

56
BotSharp.MachineLearning/WordNet/IndexWord.cs Normal file
View file

@ -0,0 +1,56 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
using System.Linq;
namespace SharpWordNet
{
/// <summary>
/// Summary description for IndexWord.
/// </summary>
public class IndexWord
{
// Properties ------------------------
public string PartOfSpeech { get; private set; }
public int[] SynsetOffsets { get; private set; }
public string Lemma { get; private set; }
public int SenseCount
{
get { return this.SynsetOffsets != null ? this.SynsetOffsets.Count() : 0; }
}
public int TagSenseCount { get; private set; }
public string[] RelationTypes { get; private set; }
// Constructors --------------------
public IndexWord(string lemma, string partOfSpeech, string[] relationTypes, int[] synsetOffsets, int tagSenseCount)
{
this.Lemma = lemma;
this.PartOfSpeech = partOfSpeech;
this.RelationTypes = relationTypes;
this.SynsetOffsets = synsetOffsets;
this.TagSenseCount = tagSenseCount;
}
}
}

74
BotSharp.MachineLearning/WordNet/Morph/AbstractDelegatingOperation.cs Normal file
View file

@ -0,0 +1,74 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the AbstractDelegatingOperation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
public abstract class AbstractDelegatingOperation : IOperation
{
private Dictionary<string, IOperation[]> mOperationSets;
public virtual void AddDelegate(string key, IOperation[] operations)
{
if (!mOperationSets.ContainsKey(key))
{
mOperationSets.Add(key, operations);
}
else
{
mOperationSets[key] = operations;
}
}
protected internal AbstractDelegatingOperation()
{
mOperationSets = new Dictionary<string, IOperation[]>();
}
//protected internal abstract AbstractDelegatingOperation getInstance(System.Collections.IDictionary params_Renamed);
protected internal virtual bool HasDelegate(string key)
{
return mOperationSets.ContainsKey(key);
}
protected internal virtual bool ExecuteDelegate(string lemma, string partOfSpeech, List<string>baseForms, string key)
{
IOperation[] operations = mOperationSets[key];
bool result = false;
for (int currentOperation = 0; currentOperation < operations.Length; currentOperation++)
{
if (operations[currentOperation].Execute(lemma, partOfSpeech, baseForms))
{
result = true;
}
}
return result;
}
#region IOperation Members
public abstract bool Execute(string lemma, string partOfSpeech, List<string> baseForms);
#endregion
}
}

67
BotSharp.MachineLearning/WordNet/Morph/DetachSuffixesOperation.cs Normal file
View file

@ -0,0 +1,67 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the DetachSuffixesOperation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
/// <summary>
/// Remove all applicable suffixes from the word(s) and do a look-up.
/// </summary>
public class DetachSuffixesOperation : AbstractDelegatingOperation
{
public const string Operations = "operations";
private Dictionary<string, string[][]> mSuffixMap;
public DetachSuffixesOperation(Dictionary<string, string[][]> suffixMap)
{
mSuffixMap = suffixMap;
}
#region IOperation Members
public override bool Execute(string lemma, string partOfSpeech, List<string> baseForms)
{
if (!mSuffixMap.ContainsKey(partOfSpeech))
{
return false;
}
string[][] suffixArray = mSuffixMap[partOfSpeech];
bool addedBaseForm = false;
for (int currentSuffix = 0; currentSuffix < suffixArray.Length; currentSuffix++)
{
if (lemma.EndsWith(suffixArray[currentSuffix][0]))
{
string stem = lemma.Substring(0, (lemma.Length - suffixArray[currentSuffix][0].Length) - (0)) + suffixArray[currentSuffix][1];
if (ExecuteDelegate(stem, partOfSpeech, baseForms, Operations))
{
addedBaseForm = true;
}
}
}
return addedBaseForm;
}
#endregion
}
}

46
BotSharp.MachineLearning/WordNet/Morph/IOperation.cs Normal file
View file

@ -0,0 +1,46 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the Operation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
public interface IOperation
{
/// <summary>
/// Execute the operation.
/// </summary>
/// <param name="lemma">
/// input lemma to look up
/// </param>
///<param name="partOfSpeech">
/// part of speech of the lemma to look up
/// </param>
/// <param name="baseForms">
/// List to which all discovered base forms should be added.
/// </param>
/// <returns>
/// True if at least one base form was discovered by the operation and
/// added to baseForms.
/// </returns>
bool Execute(string lemma, string partOfSpeech, List<string> baseForms);
}
}

57
BotSharp.MachineLearning/WordNet/Morph/LookupExceptionsOperation.cs Normal file
View file

@ -0,0 +1,57 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the LookupExceptionsOperation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
/// <summary>Lookup the word in the exceptions file of the given part-of-speech. </summary>
public class LookupExceptionsOperation : IOperation
{
private WordNetEngine mEngine;
public LookupExceptionsOperation(WordNetEngine engine)
{
mEngine = engine;
}
#region IOperation Members
public bool Execute(string lemma, string partOfSpeech, List<string> baseForms)
{
bool addedBaseForm = false;
string[] exceptionForms = mEngine.GetExceptionForms(lemma, partOfSpeech);
foreach (string exceptionForm in exceptionForms)
{
if (!baseForms.Contains(exceptionForm))
{
baseForms.Add(exceptionForm);
addedBaseForm = true;
}
}
return addedBaseForm;
}
#endregion
}
}

49
BotSharp.MachineLearning/WordNet/Morph/LookupIndexWordOperation.cs Normal file
View file

@ -0,0 +1,49 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the LookupIndexWordOperation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
public class LookupIndexWordOperation : IOperation
{
private WordNetEngine mEngine;
public LookupIndexWordOperation(WordNetEngine engine)
{
mEngine = engine;
}
#region IOperation Members
public bool Execute(string lemma, string partOfSpeech, List<string> baseForms)
{
if (!baseForms.Contains(lemma) && mEngine.GetIndexWord(lemma, partOfSpeech) != null)
{
baseForms.Add(lemma);
return true;
}
return false;
}
#endregion
}
}

181
BotSharp.MachineLearning/WordNet/Morph/TokenizerOperation.cs Normal file
View file

@ -0,0 +1,181 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the TokenizerOperation.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections.Generic;
using System.Text;
using System.Collections;
namespace SharpWordNet.Morph
{
public class TokenizerOperation : AbstractDelegatingOperation
{
/// <summary>
/// Parameter that determines the operations this operation
/// will perform on the tokens.
/// </summary>
public const string TokenOperations = "token_operations";
/// <summary>
/// Parameter that determines the operations this operation
/// will perform on the phrases.
/// </summary>
public const string PhraseOperations = "phrase_operations";
/// <summary>
/// Parameter list that determines the delimiters this
/// operation will use to concatenate tokens.
/// </summary>
public const string Delimiters = "delimiters";
private WordNetEngine mEngine;
private string[] mDelimiters;
public TokenizerOperation(WordNetEngine engine)
{
mEngine = engine;
}
public TokenizerOperation(WordNetEngine engine, string[] delimiters)
{
mEngine = engine;
mDelimiters = delimiters;
}
#region IOperation Members
public override bool Execute(string lemma, string partOfSpeech, List<string> baseForms)
{
string[] tokens = Util.Split(lemma);
List<string>[] tokenForms = new List<string>[tokens.Length];
if (!HasDelegate(TokenOperations))
{
AddDelegate(TokenOperations, new IOperation[] { new LookupIndexWordOperation(mEngine) });
}
if (!HasDelegate(PhraseOperations))
{
AddDelegate(PhraseOperations, new IOperation[] { new LookupIndexWordOperation(mEngine) });
}
for (int currentToken = 0; currentToken < tokens.Length; currentToken++)
{
tokenForms[currentToken] = new List<string>();
tokenForms[currentToken].Add(tokens[currentToken]);
ExecuteDelegate(tokens[currentToken], partOfSpeech, tokenForms[currentToken], TokenOperations);
}
bool foundForms = false;
for (int currentTokenForm = 0; currentTokenForm < tokenForms.Length; currentTokenForm++)
{
for (int tokenFormToCompare = tokenForms.Length - 1; tokenFormToCompare >= currentTokenForm; tokenFormToCompare--)
{
if (TryAllCombinations(partOfSpeech, tokenForms, currentTokenForm, tokenFormToCompare, baseForms))
{
foundForms = true;
}
}
}
return foundForms;
}
#endregion
private bool TryAllCombinations(string partOfSpeech, List<string>[] tokenForms, int startIndex, int endIndex, List<string> baseForms)
{
int length = endIndex - startIndex + 1;
int[] indexArray = new int[length];
int[] endArray = new int[length];
for (int i = 0; i < indexArray.Length; i++)
{
indexArray[i] = 0;
endArray[i] = tokenForms[startIndex + i].Count - 1;
}
bool foundForms = false;
for (; ; )
{
string[] tokens = new string[length];
for (int i = 0; i < length; i++)
{
tokens[i] = tokenForms[i + startIndex][indexArray[i]];
}
for (int i = 0; i < mDelimiters.Length; i++)
{
if (TryAllCombinations(partOfSpeech, tokens, mDelimiters[i], baseForms))
{
foundForms = true;
}
}
if (IsArrayEqual(indexArray, endArray))
{
break;
}
for (int i = length - 1; i >= 0; i--)
{
if (indexArray[i] == endArray[i])
{
indexArray[i] = 0;
}
else
{
indexArray[i]++;
break;
}
}
}
return foundForms;
}
private bool TryAllCombinations(string partOfSpeech, string[] tokens, string delimiter, List<string> baseForms)
{
BitArray bits = new BitArray(64);
int size = tokens.Length - 1;
bool foundForms = false;
do
{
string lemma = Util.GetLemma(tokens, bits, delimiter);
if (ExecuteDelegate(lemma, partOfSpeech, baseForms, PhraseOperations))
{
foundForms = true;
}
}
while (Util.Increment(bits, size));
return foundForms;
}
private bool IsArrayEqual(int[] array1, int[] array2)
{
if (array1.Length != array2.Length)
{
return false;
}
for (int i = 0; i < array1.Length; i++)
{
if (array1[i] != array2[i])
{
return false;
}
}
return true;
}
}
}

89
BotSharp.MachineLearning/WordNet/Morph/Util.cs Normal file
View file

@ -0,0 +1,89 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//This file is based on the Util.java source file found in
//the Java WordNet Library (JWNL). That source file is licensed under BSD.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
namespace SharpWordNet.Morph
{
public class Util
{
public static string GetLemma(string[] tokens, BitArray bits, string delimiter)
{
StringBuilder buf = new StringBuilder();
for (int i = 0; i < tokens.Length; i++)
{
if (i != 0 && !bits.Get(i - 1))
{
buf.Append(delimiter);
}
buf.Append(tokens[i]);
}
return buf.ToString();
}
public static bool Increment(BitArray bits, int size)
{
int i = size - 1;
while (i >= 0 && bits.Get(i))
{
bits.Set(i--, false);
}
if (i < 0)
{
return false;
}
bits.Set(i, true);
return true;
}
public static string[] Split(string str)
{
char[] chars = str.ToCharArray();
List<string> tokens = new List<string>();
StringBuilder buf = new StringBuilder();
for (int i = 0; i < chars.Length; i++)
{
if ((chars[i] >= 'a' && chars[i] <= 'z') || chars[i] == '\'')
{
buf.Append(chars[i]);
}
else
{
if (buf.Length > 0)
{
tokens.Add(buf.ToString());
buf = new StringBuilder();
}
}
}
if (buf.Length > 0)
{
tokens.Add(buf.ToString());
}
return (tokens.ToArray());
}
private Util()
{
}
}
}

85
BotSharp.MachineLearning/WordNet/Relation.cs Normal file
View file

@ -0,0 +1,85 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
namespace SharpWordNet
{
/// <summary>
/// Summary description for Relation.
/// </summary>
public class Relation
{
private WordNetEngine mWordNetEngine;
private RelationType mRelationType;
private int mTargetSynsetOffset;
private string mTargetSynsetPartOfSpeech;
private Synset mTargetSynset;
private int miSourceWord;
private int miTargetWord;
public RelationType SynsetRelationType
{
get
{
return mRelationType;
}
}
public int TargetSynsetOffset
{
get
{
return mTargetSynsetOffset;
}
}
public Synset TargetSynset
{
get
{
if (mTargetSynset == null)
{
mTargetSynset = mWordNetEngine.CreateSynset(mTargetSynsetPartOfSpeech, mTargetSynsetOffset);
}
return mTargetSynset;
}
}
private Relation()
{
}
protected internal Relation(WordNetEngine wordNetEngine, RelationType relationType, int targetSynsetOffset, string targetSynsetPartOfSpeech)
{
mWordNetEngine = wordNetEngine;
mRelationType = relationType;
mTargetSynsetOffset = targetSynsetOffset;
mTargetSynsetPartOfSpeech = targetSynsetPartOfSpeech;
}
protected internal Relation(WordNetEngine wordNetEngine, RelationType relationType, int targetSynsetOffset, string targetSynsetPartOfSpeech, int sourceWord, int targetWord) : this(wordNetEngine, relationType, targetSynsetOffset, targetSynsetPartOfSpeech)
{
miSourceWord = sourceWord;
miTargetWord = targetWord;
}
}
}

72
BotSharp.MachineLearning/WordNet/RelationType.cs Normal file
View file

@ -0,0 +1,72 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
namespace SharpWordNet
{
/// <summary>
/// Summary description for RelationType.
/// </summary>
public class RelationType
{
private string mName;
private RelationType mOpposite;
private string[] mPartsOfSpeech;
public string Name
{
get
{
return mName;
}
}
public RelationType Opposite
{
get
{
return mOpposite;
}
}
public string GetPartOfSpeech(int index)
{
return mPartsOfSpeech[index];
}
public int PartsOfSpeechCount
{
get
{
return mPartsOfSpeech.Length;
}
}
protected internal RelationType(string name, string[] partsOfSpeech)
{
mName = name;
mPartsOfSpeech = partsOfSpeech;
}
protected internal RelationType(string name, RelationType opposite, string[] partsOfSpeech)
{
mName = name;
mOpposite = opposite;
mPartsOfSpeech = partsOfSpeech;
}
}
}

113
BotSharp.MachineLearning/WordNet/Synset.cs Normal file
View file

@ -0,0 +1,113 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
namespace SharpWordNet
{
/// <summary>
/// Summary description for Synset.
/// </summary>
public class Synset
{
private int mOffset;
private string mGloss;
private string[] mWordList;
private string mLexicographerFile;
private Relation[] mRelations;
private Synset()
{
}
internal Synset(int offset, string gloss, string[] wordList, string lexicographerFile, Relation[] relations)
{
mOffset = offset;
mGloss = gloss;
mWordList = wordList;
mLexicographerFile = lexicographerFile;
mRelations = relations;
}
public int Offset
{
get
{
return mOffset;
}
}
public string Gloss
{
get
{
return mGloss;
}
}
public string GetWord(int wordIndex)
{
return mWordList[wordIndex];
}
public int WordCount
{
get
{
return mWordList.Length;
}
}
public string LexicographerFile
{
get
{
return mLexicographerFile;
}
}
public Relation GetRelation(int relationIndex)
{
return mRelations[relationIndex];
}
public int RelationCount
{
get
{
return mRelations.Length;
}
}
public override string ToString()
{
System.Text.StringBuilder oOutput = new System.Text.StringBuilder();
for (int iCurrentWord = 0; iCurrentWord < mWordList.Length; iCurrentWord++)
{
oOutput.Append(mWordList[iCurrentWord]);
if (iCurrentWord < mWordList.Length - 1)
{
oOutput.Append(", ");
}
}
oOutput.Append(" -- ").Append(mGloss);
return oOutput.ToString();
}
}
}

49
BotSharp.MachineLearning/WordNet/Tokenizer.cs Normal file
View file

@ -0,0 +1,49 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
namespace SharpWordNet
{
/// <summary>
/// Summary description for Tokenizer.
/// </summary>
public class Tokenizer
{
private readonly string[] _tokens;
int _position;
public Tokenizer(string input, params char[] separators)
{
_tokens = input.Split(separators);
_position = 0;
}
public string NextToken()
{
while (_position < _tokens.Length)
{
if ((_tokens[_position].Length > 0))
{
return _tokens[_position++];
}
_position++;
}
return null;
}
}
}

151
BotSharp.MachineLearning/WordNet/WordNetEngine.cs Normal file
View file

@ -0,0 +1,151 @@
//Copyright (C) 2006 Richard J. Northedge
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
using System;
using System.Collections.Generic;
namespace SharpWordNet
{
/// <summary>
/// Summary description for WordNetEngine.
/// </summary>
public abstract class WordNetEngine
{
private Morph.IOperation[] mDefaultOperations;
protected string[] mEmpty = new string[0];
public abstract string[] GetPartsOfSpeech();
public abstract string[] GetPartsOfSpeech(string lemma);
public abstract IndexWord[] GetAllIndexWords(string partOfSpeech);
public abstract IndexWord GetIndexWord(string lemma, string partOfSpeech);
public abstract Synset[] GetSynsets(string lemma);
public abstract Synset[] GetSynsets(string lemma, string partOfSpeech);
public abstract RelationType[] GetRelationTypes(string lemma, string partOfSpeech);
public abstract Synset GetSynset(string lemma, string partOfSpeech, int senseNumber);
public delegate void MorphologicalProcessOperation (string lemma, string partOfSpeech, List<string>baseForms);
public string[] GetBaseForms(string lemma, string partOfSpeech, MorphologicalProcessOperation morphologicalProcess)
{
var baseForms = new List<string>();
morphologicalProcess(lemma, partOfSpeech, baseForms);
return baseForms.ToArray();
}
public string[] GetBaseForms(string lemma, string partOfSpeech, Morph.IOperation[] operations)
{
var baseForms = new List<string>();
foreach (Morph.IOperation operation in operations)
{
operation.Execute(lemma, partOfSpeech, baseForms);
}
return baseForms.ToArray();
}
public string[] GetBaseForms(string lemma, string partOfSpeech)
{
if (mDefaultOperations == null)
{
var suffixMap = new Dictionary<string, string[][]>
{
{
"noun", new string[][]
{
new string[] {"s", ""}, new string[] {"ses", "s"}, new string[] {"xes", "x"},
new string[] {"zes", "z"}, new string[] {"ches", "ch"}, new string[] {"shes", "sh"},
new string[] {"men", "man"}, new string[] {"ies", "y"}
}
},
{
"verb", new string[][]
{
new string[] {"s", ""}, new string[] {"ies", "y"}, new string[] {"es", "e"},
new string[] {"es", ""}, new string[] {"ed", "e"}, new string[] {"ed", ""},
new string[] {"ing", "e"}, new string[] {"ing", ""}
}
},
{
"adjective", new string[][]
{
new string[] {"er", ""}, new string[] {"est", ""}, new string[] {"er", "e"},
new string[] {"est", "e"}
}
}
};
var tokDso = new Morph.DetachSuffixesOperation(suffixMap);
tokDso.AddDelegate(Morph.DetachSuffixesOperation.Operations, new Morph.IOperation[]
{
new Morph.LookupIndexWordOperation(this), new Morph.LookupExceptionsOperation(this)
});
var tokOp = new Morph.TokenizerOperation(this, new string[] { " ", "-" });
tokOp.AddDelegate(Morph.TokenizerOperation.TokenOperations, new Morph.IOperation[]
{
new Morph.LookupIndexWordOperation(this), new Morph.LookupExceptionsOperation(this), tokDso
});
var morphDso = new Morph.DetachSuffixesOperation(suffixMap);
morphDso.AddDelegate(Morph.DetachSuffixesOperation.Operations, new Morph.IOperation[]
{
new Morph.LookupIndexWordOperation(this), new Morph.LookupExceptionsOperation(this)
});
mDefaultOperations = new Morph.IOperation[] { new Morph.LookupExceptionsOperation(this), morphDso, tokOp };
}
return GetBaseForms(lemma, partOfSpeech, mDefaultOperations);
}
public MorphologicalProcessOperation LookupExceptionsOperation
{
get
{
return delegate(string lemma, string partOfSpeech, List<string> baseForms)
{
string[] exceptionForms = GetExceptionForms(lemma, partOfSpeech);
foreach (string exceptionForm in exceptionForms)
{
if (!baseForms.Contains(exceptionForm))
{
baseForms.Add(exceptionForm);
}
}
};
}
}
public MorphologicalProcessOperation LookupIndexWordOperation
{
get
{
return delegate(string lemma, string partOfSpeech, List<string> baseForms)
{
if (!baseForms.Contains(lemma) && GetIndexWord(lemma, partOfSpeech) != null)
{
baseForms.Add(lemma);
}
};
}
}
protected internal abstract Synset CreateSynset(string partOfSpeech, int synsetOffset);
protected internal abstract string[] GetExceptionForms(string lemma, string partOfSpeech);
}
}