TF-IDF
This commit is contained in:
Oceania2018
parent
30f0eea87c
commit
1528bd42ff
|
|
@ -1,6 +1,5 @@
|
|||
using BotSharp.NLP.Classify;
|
||||
using BotSharp.NLP.Corpus;
|
||||
using BotSharp.NLP.Models.TF_IDF;
|
||||
using BotSharp.NLP.Tokenize;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
|
|
@ -29,8 +28,8 @@ namespace BotSharp.NLP.UnitTest
|
|||
"see you Bolo",
|
||||
"byebye Haiping"
|
||||
};
|
||||
TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
|
||||
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);
|
||||
/*TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
|
||||
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);*/
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
|
|
|
|||
|
|
@ -53,6 +53,10 @@ namespace BotSharp.NLP.Classify
|
|||
|
||||
public void Train(List<Sentence> sentences, ClassifyOptions options)
|
||||
{
|
||||
var tfidf = new TFIDF();
|
||||
tfidf.Sentences = sentences;
|
||||
words = tfidf.EncodeAll();
|
||||
|
||||
var encoder = new OneHotEncoder();
|
||||
encoder.Sentences = sentences;
|
||||
words = encoder.EncodeAll();
|
||||
|
|
|
|||
|
|
@ -1,232 +0,0 @@
|
|||
using BotSharp.NLP.Tokenize;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.NLP.Models.TF_IDF
|
||||
{
|
||||
/// <summary>
|
||||
/// Copyright (c) 2018 Bo Peng
|
||||
///
|
||||
/// Permission is hereby granted, free of charge, to any person obtaining
|
||||
/// a copy of this software and associated documentation files (the
|
||||
/// "Software"), to deal in the Software without restriction, including
|
||||
/// without limitation the rights to use, copy, modify, merge, publish,
|
||||
/// distribute, sublicense, and/or sell copies of the Software, and to
|
||||
/// permit persons to whom the Software is furnished to do so, subject to
|
||||
/// the following conditions:
|
||||
///
|
||||
/// The above copyright notice and this permission notice shall be
|
||||
/// included in all copies or substantial portions of the Software.
|
||||
/// </summary>
|
||||
public class TFIDF
|
||||
{
|
||||
List<string> vocabulary { get; set; }
|
||||
|
||||
public TFIDF()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Document vocabulary, containing each word's IDF value.
|
||||
/// </summary>
|
||||
private static Dictionary<string, double> _vocabularyIDF = new Dictionary<string, double>();
|
||||
|
||||
public static List<List<double>> GetTFIDFWeightsVectors(string[] documents, int vocabularyThreshold = 1)
|
||||
{
|
||||
List<List<string>> stemmedDocs;
|
||||
List<string> vocabulary;
|
||||
// Get the vocabulary and stem the documents at the same time.
|
||||
vocabulary = GetVocabulary(documents, out stemmedDocs, vocabularyThreshold);
|
||||
if (_vocabularyIDF.Count == 0)
|
||||
{
|
||||
// Calculate the IDF for each vocabulary term.
|
||||
foreach (var term in vocabulary)
|
||||
{
|
||||
double numberOfDocsContainingTerm = stemmedDocs.Where(d => d.Contains(term)).Count();
|
||||
_vocabularyIDF[term] = Math.Log((double)stemmedDocs.Count / ((double)1 + numberOfDocsContainingTerm));
|
||||
}
|
||||
}
|
||||
// Transform each document into a vector of tfidf values.
|
||||
List<List<double>> vectors = new List<List<double>>();
|
||||
foreach (var doc in stemmedDocs)
|
||||
{
|
||||
List<double> vector = new List<double>();
|
||||
foreach (string word in doc)
|
||||
{
|
||||
double tf = doc.Where(d => d == word).Count();
|
||||
double tfidf = tf * _vocabularyIDF[word];
|
||||
vector.Add(tfidf);
|
||||
}
|
||||
vectors.Add(vector);
|
||||
}
|
||||
return vectors;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a TF*IDF array of vectors using L2-Norm.
|
||||
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
/// </summary>
|
||||
/// <param name="vectors">List<List<double>></param>
|
||||
/// <returns>List<List<double>></returns>
|
||||
public static List<List<double>> Normalize(List<List<double>> vectors)
|
||||
{
|
||||
// Normalize the vectors using L2-Norm.
|
||||
List<List<double>> normalizedVectors = new List<List<double>>();
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
var normalized = Normalize(vector);
|
||||
normalizedVectors.Add(normalized);
|
||||
}
|
||||
|
||||
return normalizedVectors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a TF*IDF vector using L2-Norm.
|
||||
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
/// </summary>
|
||||
/// <param name="vectors"> List<double> </param>
|
||||
/// <returns> List<double> </returns>
|
||||
public static List<double> Normalize(List<double> vector)
|
||||
{
|
||||
List<double> result = new List<double>();
|
||||
|
||||
double sumSquared = 0;
|
||||
foreach (var value in vector)
|
||||
{
|
||||
sumSquared += value * value;
|
||||
}
|
||||
|
||||
double SqrtSumSquared = Math.Sqrt(sumSquared);
|
||||
|
||||
foreach (var value in vector)
|
||||
{
|
||||
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
result.Add(value / SqrtSumSquared);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Saves the TFIDF vocabulary to disk.
|
||||
/// </summary>
|
||||
/// <param name="filePath">File path</param>
|
||||
public static void Save(string filePath = "vocabulary.dat")
|
||||
{
|
||||
// Save result to disk.
|
||||
using (FileStream fs = new FileStream(filePath, FileMode.Create))
|
||||
{
|
||||
BinaryFormatter formatter = new BinaryFormatter();
|
||||
formatter.Serialize(fs, _vocabularyIDF);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the TFIDF vocabulary from disk.
|
||||
/// </summary>
|
||||
/// <param name="filePath">File path</param>
|
||||
public static void Load(string filePath = "vocabulary.dat")
|
||||
{
|
||||
// Load from disk.
|
||||
using (FileStream fs = new FileStream(filePath, FileMode.Open))
|
||||
{
|
||||
BinaryFormatter formatter = new BinaryFormatter();
|
||||
_vocabularyIDF = (Dictionary<string, double>)formatter.Deserialize(fs);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses and tokenizes a list of documents, returning a vocabulary of words.
|
||||
/// </summary>
|
||||
/// <param name="docs">string[]</param>
|
||||
/// <param name="stemmedDocs">List of List of string</param>
|
||||
/// <returns>Vocabulary (list of strings)</returns>
|
||||
private static List<string> GetVocabulary(string[] docs, out List<List<string>> stemmedDocs, int vocabularyThreshold)
|
||||
{
|
||||
List<string> vocabulary = new List<string>();
|
||||
Dictionary<string, int> wordCountList = new Dictionary<string, int>();
|
||||
stemmedDocs = new List<List<string>>();
|
||||
int docIndex = 0;
|
||||
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
|
||||
{
|
||||
Pattern = RegexTokenizer.WHITE_SPACE
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
foreach (var doc in docs)
|
||||
{
|
||||
List<string> stemmedDoc = new List<string>();
|
||||
docIndex++;
|
||||
if (docIndex % 100 == 0)
|
||||
{
|
||||
Console.WriteLine("Processing " + docIndex + "/" + docs.Length);
|
||||
}
|
||||
|
||||
List<Token> tokens = tokenizer.Tokenize(doc);
|
||||
List<string> list = new List<string>();
|
||||
tokenizer.Tokenize(doc).ForEach( token => {
|
||||
list.Add(token.Text.ToLower());
|
||||
});
|
||||
string[] parts2 = list.ToArray();
|
||||
//string[] parts2 = Tokenize(doc);
|
||||
List<string> words = new List<string>();
|
||||
foreach (string part in parts2)
|
||||
{
|
||||
// Strip non-alphanumeric characters.
|
||||
string stripped = Regex.Replace(part, "[^a-zA-Z0-9]", "");
|
||||
try
|
||||
{
|
||||
var english = new EnglishWord(stripped);
|
||||
string stem = english.Stem;
|
||||
words.Add(stem);
|
||||
|
||||
if (stem.Length > 0)
|
||||
{
|
||||
// Build the word count list.
|
||||
if (wordCountList.ContainsKey(stem))
|
||||
{
|
||||
wordCountList[stem]++;
|
||||
}
|
||||
else
|
||||
{
|
||||
wordCountList.Add(stem, 0);
|
||||
}
|
||||
stemmedDoc.Add(stem);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
stemmedDocs.Add(stemmedDoc);
|
||||
}
|
||||
// Get the top words.
|
||||
var vocabList = wordCountList.Where(w => w.Value >= vocabularyThreshold);
|
||||
foreach (var item in vocabList)
|
||||
{
|
||||
vocabulary.Add(item.Key);
|
||||
}
|
||||
return vocabulary;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
public class EnglishWord
|
||||
{
|
||||
public EnglishWord(string input)
|
||||
{
|
||||
this.Original = input;
|
||||
this.Stem = input;
|
||||
this.Length = input.Length;
|
||||
}
|
||||
|
||||
public string Stem { get; set; }
|
||||
public string Original { get; }
|
||||
public int Length { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace BotSharp.NLP.Models.TF_IDF
|
||||
{
|
||||
/// <summary>
|
||||
/// Copyright (c) 2018 Bo Peng
|
||||
///
|
||||
/// Permission is hereby granted, free of charge, to any person obtaining
|
||||
/// a copy of this software and associated documentation files (the
|
||||
/// "Software"), to deal in the Software without restriction, including
|
||||
/// without limitation the rights to use, copy, modify, merge, publish,
|
||||
/// distribute, sublicense, and/or sell copies of the Software, and to
|
||||
/// permit persons to whom the Software is furnished to do so, subject to
|
||||
/// the following conditions:
|
||||
///
|
||||
/// The above copyright notice and this permission notice shall be
|
||||
/// included in all copies or substantial portions of the Software.
|
||||
/// </summary>
|
||||
public class TFIDFGenerator
|
||||
{
|
||||
public List<List<double>> TFIDFWeightVectorsForSentences(string[]documents)
|
||||
{
|
||||
List<List<double>> res = TFIDF.GetTFIDFWeightsVectors(documents, 0);
|
||||
res = TFIDF.Normalize(res);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
}
|
||||
148
BotSharp.NLP/Txt2Vec/TFIDF.cs
Normal file
148
BotSharp.NLP/Txt2Vec/TFIDF.cs
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
/// <summary>
|
||||
/// Copyright (c) 2018 Bo Peng
|
||||
///
|
||||
/// Permission is hereby granted, free of charge, to any person obtaining
|
||||
/// a copy of this software and associated documentation files (the
|
||||
/// "Software"), to deal in the Software without restriction, including
|
||||
/// without limitation the rights to use, copy, modify, merge, publish,
|
||||
/// distribute, sublicense, and/or sell copies of the Software, and to
|
||||
/// permit persons to whom the Software is furnished to do so, subject to
|
||||
/// the following conditions:
|
||||
///
|
||||
/// The above copyright notice and this permission notice shall be
|
||||
/// included in all copies or substantial portions of the Software.
|
||||
/// </summary>
|
||||
///
|
||||
using BotSharp.NLP.Tokenize;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.NLP.Txt2Vec
|
||||
{
|
||||
public class TFIDF
|
||||
{
|
||||
public List<Sentence> Sentences { get; set; }
|
||||
|
||||
public List<string> Words { get; set; }
|
||||
|
||||
public void Encode(Sentence sentence)
|
||||
{
|
||||
InitDictionary();
|
||||
|
||||
// var featureSets = Sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
|
||||
|
||||
var labelDist = Sentences.Select(x => x.Label).Distinct().ToList();
|
||||
|
||||
labelDist.ForEach(label =>
|
||||
{
|
||||
// https://zhuanlan.zhihu.com/p/31197209
|
||||
// calculate TF
|
||||
// all words in the article
|
||||
List<string> words = new List<string>();
|
||||
Sentences.Where(x => x.Label == label).ToList().ForEach(sent =>
|
||||
{
|
||||
words.AddRange(sent.Words.Select(w => w.Text));
|
||||
});
|
||||
|
||||
List<Tuple<string, double>> tfs = new List<Tuple<string, double>>();
|
||||
words.Distinct().ToList().ForEach(w =>
|
||||
{
|
||||
// TF
|
||||
int c1 = words.Count(x => x == w);
|
||||
double tf = (c1 + 1.0) / words.Count();
|
||||
|
||||
// IDF
|
||||
var sents = Sentences.Where(s => s.Words.Select(x => x.Text).Contains(w)).ToList();
|
||||
double idf = Math.Log(Sentences.Count / (sents.Count() + 1.0));
|
||||
|
||||
tfs.Add(new Tuple<string, double>(w, tf * idf));
|
||||
});
|
||||
|
||||
tfs = tfs.OrderByDescending(x => x.Item2).Take(words.Count / 10).ToList();
|
||||
});
|
||||
|
||||
|
||||
|
||||
sentence.Words.ForEach(w =>
|
||||
{
|
||||
int index = Words.IndexOf(w.Text.ToLower());
|
||||
});
|
||||
}
|
||||
|
||||
public List<string> EncodeAll()
|
||||
{
|
||||
InitDictionary();
|
||||
|
||||
Sentences.ForEach(sent => Encode(sent));
|
||||
//Parallel.ForEach(Sentences, sent => Encode(sent));
|
||||
|
||||
return Words;
|
||||
}
|
||||
|
||||
private List<string> InitDictionary()
|
||||
{
|
||||
if (Words == null)
|
||||
{
|
||||
Words = new List<string>();
|
||||
Sentences.ForEach(x =>
|
||||
{
|
||||
Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
|
||||
});
|
||||
Words = Words.Distinct().OrderBy(x => x).ToList();
|
||||
}
|
||||
|
||||
return Words;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a TF*IDF array of vectors using L2-Norm.
|
||||
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
/// </summary>
|
||||
/// <param name="vectors">List<List<double>></param>
|
||||
/// <returns>List<List<double>></returns>
|
||||
public static List<List<double>> Normalize(List<List<double>> vectors)
|
||||
{
|
||||
// Normalize the vectors using L2-Norm.
|
||||
List<List<double>> normalizedVectors = new List<List<double>>();
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
var normalized = Normalize(vector);
|
||||
normalizedVectors.Add(normalized);
|
||||
}
|
||||
|
||||
return normalizedVectors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a TF*IDF vector using L2-Norm.
|
||||
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
/// </summary>
|
||||
/// <param name="vectors"> List<double> </param>
|
||||
/// <returns> List<double> </returns>
|
||||
public static List<double> Normalize(List<double> vector)
|
||||
{
|
||||
List<double> result = new List<double>();
|
||||
|
||||
double sumSquared = 0;
|
||||
foreach (var value in vector)
|
||||
{
|
||||
sumSquared += value * value;
|
||||
}
|
||||
|
||||
double SqrtSumSquared = Math.Sqrt(sumSquared);
|
||||
|
||||
foreach (var value in vector)
|
||||
{
|
||||
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
|
||||
result.Add(value / SqrtSumSquared);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ using System.Text;
|
|||
using System.Threading.Tasks;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using BotSharp.NLP.Models.TF_IDF;
|
||||
//using AdvUtils;
|
||||
|
||||
namespace Txt2Vec
|
||||
|
|
@ -37,8 +36,8 @@ namespace Txt2Vec
|
|||
public List<Vec> Sentence2Vec(List<string> sentences, WeightingScheme weightingScheme = WeightingScheme.AVG)
|
||||
{
|
||||
// Inplementing TF-IDF
|
||||
TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
|
||||
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray());
|
||||
// TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
|
||||
List<List<double>> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray());
|
||||
|
||||
List<List<Vec>> matixList = new List<List<Vec>>();
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue