512 lines
20 KiB
C#
512 lines
20 KiB
C#
//Copyright (C) 2006 Richard J. Northedge
|
|
//
|
|
// This library is free software; you can redistribute it and/or
|
|
// modify it under the terms of the GNU Lesser General Public
|
|
// License as published by the Free Software Foundation; either
|
|
// version 2.1 of the License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
using System;
|
|
using System.IO;
|
|
using System.Collections.Generic;
|
|
|
|
namespace BotSharp.MachineLearning
|
|
{
|
|
/// <summary>
|
|
/// Summary description for DataFileEngine.
|
|
/// </summary>
|
|
public class DataFileEngine : WordNetEngine
|
|
{
|
|
private readonly string _dataFolder;
|
|
private readonly Dictionary<string, PosDataFileSet> _dataFileDictionary;
|
|
private string[] _lexicographerFiles;
|
|
private Dictionary<string, RelationType> _relationTypeDictionary;
|
|
|
|
|
|
// Public Methods (class specific) ------------------
|
|
public string DataFolder
|
|
{
|
|
get
|
|
{
|
|
return _dataFolder;
|
|
}
|
|
}
|
|
|
|
public DataFileEngine(string dataFolder)
|
|
{
|
|
_dataFolder = dataFolder;
|
|
|
|
_dataFileDictionary = new Dictionary<string, PosDataFileSet>(4)
|
|
{
|
|
{"noun", new PosDataFileSet(dataFolder, "noun")},
|
|
{"verb", new PosDataFileSet(dataFolder, "verb")},
|
|
{"adjective", new PosDataFileSet(dataFolder, "adj")},
|
|
{"adverb", new PosDataFileSet(dataFolder, "adv")}
|
|
};
|
|
|
|
InitializeLexicographerFiles();
|
|
|
|
InitializeRelationTypes();
|
|
}
|
|
|
|
|
|
// abstract methods implementation ------------------
|
|
|
|
public override string[] GetPartsOfSpeech()
|
|
{
|
|
return new List<string>(_dataFileDictionary.Keys).ToArray();
|
|
}
|
|
|
|
public override string[] GetPartsOfSpeech(string lemma)
|
|
{
|
|
var partsOfSpeech = new List<string>();
|
|
foreach (string partOfSpeech in _dataFileDictionary.Keys)
|
|
{
|
|
if (BinarySearch(lemma, _dataFileDictionary[partOfSpeech].IndexFile) != null)
|
|
{
|
|
partsOfSpeech.Add(partOfSpeech);
|
|
}
|
|
}
|
|
return partsOfSpeech.ToArray();
|
|
}
|
|
|
|
public override IndexWord[] GetAllIndexWords(string partOfSpeech)
|
|
{
|
|
StreamReader searchFile = _dataFileDictionary[partOfSpeech].IndexFile;
|
|
string line;
|
|
string space = " ";
|
|
var indexWords = new List<IndexWord>();
|
|
searchFile.DiscardBufferedData();
|
|
searchFile.BaseStream.Position = 0;
|
|
while (!searchFile.EndOfStream)
|
|
{
|
|
line = searchFile.ReadLine();
|
|
if (!line.StartsWith(space))
|
|
{
|
|
indexWords.Add(CreateIndexWord(partOfSpeech, line));
|
|
}
|
|
}
|
|
return indexWords.ToArray();
|
|
}
|
|
|
|
public override IndexWord GetIndexWord(string lemma, string partOfSpeech)
|
|
{
|
|
string line = BinarySearch(lemma, _dataFileDictionary[partOfSpeech].IndexFile);
|
|
if (line != null)
|
|
{
|
|
return CreateIndexWord(partOfSpeech, line);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public override Synset[] GetSynsets(string lemma)
|
|
{
|
|
var synsets = new List<Synset>();
|
|
|
|
foreach (string partOfSpeech in _dataFileDictionary.Keys)
|
|
{
|
|
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
|
|
|
|
if (indexWord != null)
|
|
{
|
|
foreach (int synsetOffset in indexWord.SynsetOffsets)
|
|
{
|
|
Synset synset = CreateSynset(partOfSpeech, synsetOffset);
|
|
synsets.Add(synset);
|
|
}
|
|
}
|
|
}
|
|
return synsets.ToArray();
|
|
}
|
|
|
|
public override Synset[] GetSynsets(string lemma, string partOfSpeech)
|
|
{
|
|
var synsets = new List<Synset>();
|
|
|
|
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
|
|
|
|
if (indexWord != null)
|
|
{
|
|
foreach (int synsetOffset in indexWord.SynsetOffsets)
|
|
{
|
|
Synset synset = CreateSynset(partOfSpeech, synsetOffset);
|
|
synsets.Add(synset);
|
|
}
|
|
}
|
|
|
|
return synsets.ToArray();
|
|
}
|
|
|
|
public override RelationType[] GetRelationTypes(string lemma, string partOfSpeech)
|
|
{
|
|
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
|
|
|
|
if (indexWord != null)
|
|
{
|
|
if (indexWord.RelationTypes != null)
|
|
{
|
|
int relationTypeCount = indexWord.RelationTypes.Length;
|
|
var relationTypes = new RelationType[relationTypeCount];
|
|
for (int currentRelationType = 0; currentRelationType < relationTypeCount; currentRelationType++)
|
|
{
|
|
relationTypes[currentRelationType] = _relationTypeDictionary[indexWord.RelationTypes[currentRelationType]];
|
|
}
|
|
return relationTypes;
|
|
}
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public override Synset GetSynset(string lemma, string partOfSpeech, int senseNumber)
|
|
{
|
|
if (senseNumber < 1)
|
|
{
|
|
throw new ArgumentOutOfRangeException("senseNumber", senseNumber, "cannot be less than 1");
|
|
}
|
|
|
|
IndexWord indexWord = GetIndexWord(lemma, partOfSpeech);
|
|
|
|
if (indexWord != null)
|
|
{
|
|
if (senseNumber > (indexWord.SynsetOffsets.Length + 1))
|
|
{
|
|
return (null);
|
|
}
|
|
Synset synset = CreateSynset(partOfSpeech, indexWord.SynsetOffsets[senseNumber - 1]);
|
|
return (synset);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
|
|
// Private Methods----------------------------------
|
|
|
|
private string BinarySearch(string searchKey, StreamReader searchFile)
|
|
{
|
|
if (searchKey.Length == 0)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
int c,n;
|
|
long top,bot,mid,diff;
|
|
string line,key;
|
|
diff = 666;
|
|
line = "";
|
|
bot = searchFile.BaseStream.Seek(0, SeekOrigin.End);
|
|
top = 0;
|
|
mid = (bot-top)/2;
|
|
|
|
do
|
|
{
|
|
searchFile.DiscardBufferedData();
|
|
searchFile.BaseStream.Position = mid - 1;
|
|
if (mid != 1)
|
|
{
|
|
while ((c = searchFile.Read()) != '\n' && c != -1) { }
|
|
}
|
|
line = searchFile.ReadLine();
|
|
if (line == null)
|
|
{
|
|
return null;
|
|
}
|
|
n = line.IndexOf(' ');
|
|
key = line.Substring(0,n);
|
|
key=key.Replace("-"," ").Replace("_"," ");
|
|
if (string.CompareOrdinal(key, searchKey) < 0)
|
|
{
|
|
top = mid;
|
|
diff = (bot - top)/2;
|
|
mid = top + diff;
|
|
}
|
|
if (string.CompareOrdinal(key, searchKey) > 0)
|
|
{
|
|
bot = mid;
|
|
diff = (bot - top)/2;
|
|
mid = top + diff;
|
|
}
|
|
} while (key!=searchKey && diff!=0);
|
|
|
|
if (key == searchKey)
|
|
{
|
|
return line;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private IndexWord CreateIndexWord(string partOfSpeech, string line)
|
|
{
|
|
var tokenizer = new Tokenizer(line);
|
|
string word = tokenizer.NextToken().Replace('_', ' ');
|
|
string redundantPartOfSpeech = tokenizer.NextToken();
|
|
int senseCount = int.Parse(tokenizer.NextToken());
|
|
|
|
int relationTypeCount = int.Parse(tokenizer.NextToken());
|
|
string[] relationTypes = null;
|
|
if (relationTypeCount > 0)
|
|
{
|
|
relationTypes = new string[relationTypeCount];
|
|
for (int currentRelationType = 0; currentRelationType < relationTypeCount; currentRelationType++)
|
|
{
|
|
relationTypes[currentRelationType] = tokenizer.NextToken();
|
|
}
|
|
}
|
|
int redundantSenseCount = int.Parse(tokenizer.NextToken());
|
|
int tagSenseCount = int.Parse(tokenizer.NextToken());
|
|
|
|
int[] synsetOffsets = null;
|
|
if (senseCount > 0)
|
|
{
|
|
synsetOffsets = new int[senseCount];
|
|
for (int currentOffset = 0; currentOffset < senseCount; currentOffset++)
|
|
{
|
|
synsetOffsets[currentOffset] = int.Parse(tokenizer.NextToken());
|
|
}
|
|
}
|
|
return new IndexWord(word, partOfSpeech, relationTypes, synsetOffsets, tagSenseCount);
|
|
}
|
|
|
|
protected internal override Synset CreateSynset(string partOfSpeech, int synsetOffset)
|
|
{
|
|
StreamReader dataFile = _dataFileDictionary[partOfSpeech].DataFile;
|
|
dataFile.DiscardBufferedData();
|
|
dataFile.BaseStream.Seek(synsetOffset, SeekOrigin.Begin);
|
|
string record = dataFile.ReadLine();
|
|
|
|
var tokenizer = new Tokenizer(record);
|
|
var nextToken = tokenizer.NextToken();
|
|
int offset = int.Parse(nextToken);
|
|
|
|
|
|
var nt = int.Parse(tokenizer.NextToken());
|
|
string lexicographerFile = _lexicographerFiles[nt];
|
|
string synsetType = tokenizer.NextToken();
|
|
int wordCount = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
|
|
|
|
var words = new string[wordCount];
|
|
for (int iCurrentWord = 0; iCurrentWord < wordCount; iCurrentWord++)
|
|
{
|
|
words[iCurrentWord] = tokenizer.NextToken().Replace("_", " ");
|
|
int uniqueID = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
|
|
}
|
|
|
|
int relationCount = int.Parse(tokenizer.NextToken());
|
|
var relations = new Relation[relationCount];
|
|
for (int currentRelation = 0; currentRelation < relationCount; currentRelation++)
|
|
{
|
|
string relationTypeKey = tokenizer.NextToken();
|
|
// if (fpos.name=="adj" && sstype==AdjSynSetType.DontKnow)
|
|
// {
|
|
// if (ptrs[j].ptp.mnemonic=="ANTPTR")
|
|
// sstype = AdjSynSetType.DirectAnt;
|
|
// else if (ptrs[j].ptp.mnemonic=="PERTPTR")
|
|
// sstype = AdjSynSetType.Pertainym;
|
|
// }
|
|
int targetSynsetOffset = int.Parse(tokenizer.NextToken());
|
|
string targetPartOfSpeech = tokenizer.NextToken();
|
|
switch (targetPartOfSpeech)
|
|
{
|
|
case "n":
|
|
targetPartOfSpeech = "noun";
|
|
break;
|
|
case "v":
|
|
targetPartOfSpeech = "verb";
|
|
break;
|
|
case "a":
|
|
case "s":
|
|
targetPartOfSpeech = "adjective";
|
|
break;
|
|
case "r":
|
|
targetPartOfSpeech = "adverb";
|
|
break;
|
|
}
|
|
|
|
int sourceTarget = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
|
|
if (sourceTarget == 0)
|
|
{
|
|
relations[currentRelation] = new Relation(this, (RelationType)_relationTypeDictionary[relationTypeKey], targetSynsetOffset, targetPartOfSpeech);
|
|
}
|
|
else
|
|
{
|
|
int sourceWord = sourceTarget >> 8;
|
|
int targetWord = sourceTarget & 0xff;
|
|
relations[currentRelation] = new Relation(this, (RelationType)_relationTypeDictionary[relationTypeKey], targetSynsetOffset, targetPartOfSpeech, sourceWord, targetWord);
|
|
}
|
|
}
|
|
string frameData = tokenizer.NextToken();
|
|
if (frameData != "|")
|
|
{
|
|
int frameCount = int.Parse(frameData);
|
|
for (int currentFrame = 0; currentFrame < frameCount; currentFrame++)
|
|
{
|
|
frameData = tokenizer.NextToken(); // +
|
|
int frameNumber = int.Parse(tokenizer.NextToken());
|
|
int wordID = int.Parse(tokenizer.NextToken(), System.Globalization.NumberStyles.HexNumber);
|
|
}
|
|
frameData = tokenizer.NextToken();
|
|
}
|
|
string gloss = record.Substring(record.IndexOf('|') + 1);
|
|
|
|
var synset = new Synset(synsetOffset, gloss, words, lexicographerFile, relations);
|
|
return synset;
|
|
}
|
|
|
|
protected internal override string[] GetExceptionForms(string lemma, string partOfSpeech)
|
|
{
|
|
string line = BinarySearch(lemma, _dataFileDictionary[partOfSpeech].ExceptionFile);
|
|
if (line != null)
|
|
{
|
|
var exceptionForms = new List<string>();
|
|
var tokenizer = new Tokenizer(line);
|
|
string skipWord = tokenizer.NextToken();
|
|
string word = tokenizer.NextToken();
|
|
while (word != null)
|
|
{
|
|
exceptionForms.Add(word);
|
|
word = tokenizer.NextToken();
|
|
}
|
|
return exceptionForms.ToArray();
|
|
}
|
|
return mEmpty;
|
|
}
|
|
|
|
private void InitializeLexicographerFiles()
|
|
{
|
|
_lexicographerFiles = new string[45];
|
|
|
|
_lexicographerFiles[0] = "adj.all - all adjective clusters";
|
|
_lexicographerFiles[1] = "adj.pert - relational adjectives (pertainyms)";
|
|
_lexicographerFiles[2] = "adv.all - all adverbs";
|
|
_lexicographerFiles[3] = "noun.Tops - unique beginners for nouns";
|
|
_lexicographerFiles[4] = "noun.act - nouns denoting acts or actions";
|
|
_lexicographerFiles[5] = "noun.animal - nouns denoting animals";
|
|
_lexicographerFiles[6] = "noun.artifact - nouns denoting man-made objects";
|
|
_lexicographerFiles[7] = "noun.attribute - nouns denoting attributes of people and objects";
|
|
_lexicographerFiles[8] = "noun.body - nouns denoting body parts";
|
|
_lexicographerFiles[9] = "noun.cognition - nouns denoting cognitive processes and contents";
|
|
_lexicographerFiles[10] = "noun.communication - nouns denoting communicative processes and contents";
|
|
_lexicographerFiles[11] = "noun.event - nouns denoting natural events";
|
|
_lexicographerFiles[12] = "noun.feeling - nouns denoting feelings and emotions";
|
|
_lexicographerFiles[13] = "noun.food - nouns denoting foods and drinks";
|
|
_lexicographerFiles[14] = "noun.group - nouns denoting groupings of people or objects";
|
|
_lexicographerFiles[15] = "noun.location - nouns denoting spatial position";
|
|
_lexicographerFiles[16] = "noun.motive - nouns denoting goals";
|
|
_lexicographerFiles[17] = "noun.object - nouns denoting natural objects (not man-made)";
|
|
_lexicographerFiles[18] = "noun.person - nouns denoting people";
|
|
_lexicographerFiles[19] = "noun.phenomenon - nouns denoting natural phenomena";
|
|
_lexicographerFiles[20] = "noun.plant - nouns denoting plants";
|
|
_lexicographerFiles[21] = "noun.possession - nouns denoting possession and transfer of possession";
|
|
_lexicographerFiles[22] = "noun.process - nouns denoting natural processes";
|
|
_lexicographerFiles[23] = "noun.quantity - nouns denoting quantities and units of measure";
|
|
_lexicographerFiles[24] = "noun.relation - nouns denoting relations between people or things or ideas";
|
|
_lexicographerFiles[25] = "noun.shape - nouns denoting two and three dimensional shapes";
|
|
_lexicographerFiles[26] = "noun.state - nouns denoting stable states of affairs";
|
|
_lexicographerFiles[27] = "noun.substance - nouns denoting substances";
|
|
_lexicographerFiles[28] = "noun.time - nouns denoting time and temporal relations";
|
|
_lexicographerFiles[29] = "verb.body - verbs of grooming, dressing and bodily care";
|
|
_lexicographerFiles[30] = "verb.change - verbs of size, temperature change, intensifying, etc.";
|
|
_lexicographerFiles[31] = "verb.cognition - verbs of thinking, judging, analyzing, doubting";
|
|
_lexicographerFiles[32] = "verb.communication - verbs of telling, asking, ordering, singing";
|
|
_lexicographerFiles[33] = "verb.competition - verbs of fighting, athletic activities";
|
|
_lexicographerFiles[34] = "verb.consumption - verbs of eating and drinking";
|
|
_lexicographerFiles[35] = "verb.contact - verbs of touching, hitting, tying, digging";
|
|
_lexicographerFiles[36] = "verb.creation - verbs of sewing, baking, painting, performing";
|
|
_lexicographerFiles[37] = "verb.emotion - verbs of feeling";
|
|
_lexicographerFiles[38] = "verb.motion - verbs of walking, flying, swimming";
|
|
_lexicographerFiles[39] = "verb.perception - verbs of seeing, hearing, feeling";
|
|
_lexicographerFiles[40] = "verb.possession - verbs of buying, selling, owning";
|
|
_lexicographerFiles[41] = "verb.social - verbs of political and social activities and events";
|
|
_lexicographerFiles[42] = "verb.stative - verbs of being, having, spatial relations";
|
|
_lexicographerFiles[43] = "verb.weather - verbs of raining, snowing, thawing, thundering";
|
|
_lexicographerFiles[44] = "adj.ppl - participial adjectives";
|
|
|
|
}
|
|
|
|
private void InitializeRelationTypes()
|
|
{
|
|
_relationTypeDictionary = new Dictionary<string, RelationType>(30)
|
|
{
|
|
{"!", new RelationType("Antonym", new string[] {"noun", "verb", "adjective", "adverb"})},
|
|
{"@", new RelationType("Hypernym", new string[] {"noun", "verb"})},
|
|
{"@i", new RelationType("Instance Hypernym", new string[] {"noun"})},
|
|
{"~", new RelationType("Hyponym", new string[] {"noun", "verb"})},
|
|
{"~i", new RelationType("Instance Hyponym", new string[] {"noun"})},
|
|
{"#m", new RelationType("Member holonym", new string[] {"noun"})},
|
|
{"#s", new RelationType("Substance holonym", new string[] {"noun"})},
|
|
{"#p", new RelationType("Part holonym", new string[] {"noun"})},
|
|
{"%m", new RelationType("Member meronym", new string[] {"noun"})},
|
|
{"%s", new RelationType("Substance meronym", new string[] {"noun"})},
|
|
{"%p", new RelationType("Part meronym", new string[] {"noun"})},
|
|
{"=", new RelationType("Attribute", new string[] {"noun", "adjective"})},
|
|
{"+", new RelationType("Derivationally related form", new string[] {"noun", "verb"})},
|
|
{";c", new RelationType("Domain of synset - TOPIC", new string[] {"noun", "verb", "adjective", "adverb"})},
|
|
{"-c", new RelationType("Member of this domain - TOPIC", new string[] {"noun"})},
|
|
{";r", new RelationType("Domain of synset - REGION", new string[] {"noun", "verb", "adjective", "adverb"})},
|
|
{"-r", new RelationType("Member of this domain - REGION", new string[] {"noun"})},
|
|
{";u", new RelationType("Domain of synset - USAGE", new string[] {"noun", "verb", "adjective", "adverb"})},
|
|
{"-u", new RelationType("Member of this domain - USAGE", new string[] {"noun"})},
|
|
{"*", new RelationType("Entailment", new string[] {"verb"})},
|
|
{">", new RelationType("Cause", new string[] {"verb"})},
|
|
{"^", new RelationType("Also see", new string[] {"verb", "adjective"})},
|
|
{"$", new RelationType("Verb Group", new string[] {"verb"})},
|
|
{"&", new RelationType("Similar to", new string[] {"adjective"})},
|
|
{"<", new RelationType("Participle of verb", new string[] {"adjective"})},
|
|
{@"\", new RelationType("Pertainym", new string[] {"adjective", "adverb"})}
|
|
};
|
|
|
|
//moRelationTypeDictionary.Add(";", new RelationType("Domain of synset", new string[] {"noun", "verb", "adjective", "adverb"}));
|
|
//moRelationTypeDictionary.Add("-", new RelationType("Member of this domain", new string[] {"noun"}));
|
|
|
|
}
|
|
|
|
private class PosDataFileSet
|
|
{
|
|
private readonly StreamReader _indexFile;
|
|
private readonly StreamReader _dataFile;
|
|
private readonly StreamReader _exceptionFile;
|
|
|
|
public StreamReader IndexFile
|
|
{
|
|
get
|
|
{
|
|
return _indexFile;
|
|
}
|
|
}
|
|
|
|
public StreamReader DataFile
|
|
{
|
|
get
|
|
{
|
|
return _dataFile;
|
|
}
|
|
}
|
|
|
|
public StreamReader ExceptionFile
|
|
{
|
|
get
|
|
{
|
|
return _exceptionFile;
|
|
}
|
|
}
|
|
|
|
public PosDataFileSet(string dataFolder, string partOfSpeech)
|
|
{
|
|
_indexFile = new StreamReader(Path.Combine(dataFolder, "index." + partOfSpeech));
|
|
_dataFile = new StreamReader(Path.Combine(dataFolder, "data." + partOfSpeech));
|
|
_exceptionFile = new StreamReader(Path.Combine(dataFolder, partOfSpeech + ".exc"));
|
|
}
|
|
}
|
|
|
|
|
|
}
|
|
}
|