SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

extractor key words from document.

Browse source
This commit is contained in:
Esther2013 2018-09-12 23:33:34 -05:00
parent 1528bd42ff
commit a8c5242c7b
7 changed files with 178 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -22,6 +22,7 @@ using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Extensions;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using BotSharp.NLP.Featuring;
using BotSharp.NLP.Txt2Vec;
using Newtonsoft.Json;
using System;
@ -53,10 +54,11 @@ namespace BotSharp.NLP.Classify
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
var tfidf = new TFIDF();
var tfidf = new TfIdfFeatureExtractor();
tfidf.Sentences = sentences;
words = tfidf.EncodeAll();
tfidf.CalBasedOnCategory();
var keyWords = tfidf.Features();
string keywords2 = String.Join(",", keyWords.ToArray());
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
words = encoder.EncodeAll();

10
BotSharp.NLP/Featuring/IFeatureExtractor.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.NLP.Featuring
{
public interface IFeatureExtractor
{
}
}

155
BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs Normal file
View file

@ -0,0 +1,155 @@
/*
* BotSharp.NLP Library
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Featuring
{
public class TfIdfFeatureExtractor : IFeatureExtractor
{
public List<Sentence> Sentences { get; set; }
private List<Tuple<String, double>> tfs;
private List<string> Categories { get; set; }
public void Extract(Sentence sentence)
{
}
public List<string> Features()
{
var tfs2 = tfs.OrderByDescending(x => x.Item2)
.Select(x => x.Item1)
.Distinct()
.Take(Sentences.Count / Categories.Count)
.ToList();
return tfs2;
}
public void CalBasedOnSentence()
{
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
tfs = new List<Tuple<String, double>>();
Sentences.ForEach(sent =>
{
sent.Words.ForEach(word =>
{
// TF
int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
double tf = (c1 + 1.0) / sent.Words.Count();
// IDF
var c2 = Sentences.Count(s => s.Words.Select(x => x.Lemma).Contains(word.Lemma));
double idf = Math.Log(Sentences.Count / (c2 + 1.0));
word.Vector = tf * idf;
tfs.Add(new Tuple<string, double>(word.Lemma, word.Vector));
});
});
}
public void CalBasedOnCategory()
{
tfs = new List<Tuple<String, double>>();
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
Categories.ForEach(label =>
{
var allTokens = new List<Token>();
Sentences.Where(x => x.Label == label)
.ToList()
.ForEach(s => allTokens.AddRange(s.Words));
allTokens.Select(x => x.Lemma).Distinct()
.ToList()
.ForEach(word =>
{
// TF
int c1 = allTokens.Count(x => x.Lemma == word);
double tf = (c1 + 1.0) / allTokens.Count();
// IDF
var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
.GroupBy(x => x.Label).Count();
double idf = Math.Log(Categories.Count / (c2 + 1.0));
tfs.Add(new Tuple<string, double>(word, tf * idf));
});
});
}
/// <summary>
/// Normalizes a TF*IDF array of vectors using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors">List<List<double>></param>
/// <returns>List<List<double>></returns>
public static List<List<double>> Normalize(List<List<double>> vectors)
{
// Normalize the vectors using L2-Norm.
List<List<double>> normalizedVectors = new List<List<double>>();
foreach (var vector in vectors)
{
var normalized = Normalize(vector);
normalizedVectors.Add(normalized);
}
return normalizedVectors;
}
/// <summary>
/// Normalizes a TF*IDF vector using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors"> List<double> </param>
/// <returns> List<double> </returns>
public static List<double> Normalize(List<double> vector)
{
List<double> result = new List<double>();
double sumSquared = 0;
foreach (var value in vector)
{
sumSquared += value * value;
}
double SqrtSumSquared = Math.Sqrt(sumSquared);
foreach (var value in vector)
{
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
result.Add(value / SqrtSumSquared);
}
return result;
}
}
}

2
BotSharp.NLP/Tokenize/Token.cs
View file

@ -67,5 +67,7 @@ namespace BotSharp.NLP.Tokenize
{
return $"{Text} {Start} {Pos}";
}
public double Vector { get; set; }
}
}

5
BotSharp.NLP/Tokenize/TokenizerFactory.cs
View file

@ -29,7 +29,9 @@ namespace BotSharp.NLP.Tokenize
public List<Token> Tokenize(string sentence)
{
return _tokenizer.Tokenize(sentence, _options);
var tokens = _tokenizer.Tokenize(sentence, _options);
tokens.ForEach(x => x.Lemma = x.Text.ToLower());
return tokens;
}
public List<Sentence> Tokenize(List<String> sentences)
@ -39,6 +41,7 @@ namespace BotSharp.NLP.Tokenize
Parallel.ForEach(sents, (sentence) =>
{
sentence.Words = Tokenize(sentence.Text);
sentence.Words.ForEach(x => x.Lemma = x.Text.ToLower());
});
return sents;

9
BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
View file

@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec
sentence.Words.ForEach(w =>
{
int index = Words.IndexOf(w.Text.ToLower());
int index = Words.IndexOf(w.Lemma.ToLower());
if(index > 0)
{
vector[index] = 1;
@ -49,12 +49,7 @@ namespace BotSharp.NLP.Txt2Vec
{
if (Words == null)
{
Words = new List<string>();
Sentences.ForEach(x =>
{
Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
});
Words = Words.Distinct().OrderBy(x => x).ToList();
// Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList();
}
return Words;

148
BotSharp.NLP/Txt2Vec/TFIDF.cs
View file

@ -1,148 +0,0 @@
/// <summary>
/// Copyright (c) 2018 Bo Peng
///
/// Permission is hereby granted, free of charge, to any person obtaining
/// a copy of this software and associated documentation files (the
/// "Software"), to deal in the Software without restriction, including
/// without limitation the rights to use, copy, modify, merge, publish,
/// distribute, sublicense, and/or sell copies of the Software, and to
/// permit persons to whom the Software is furnished to do so, subject to
/// the following conditions:
///
/// The above copyright notice and this permission notice shall be
/// included in all copies or substantial portions of the Software.
/// </summary>
///
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Txt2Vec
{
public class TFIDF
{
public List<Sentence> Sentences { get; set; }
public List<string> Words { get; set; }
public void Encode(Sentence sentence)
{
InitDictionary();
// var featureSets = Sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
var labelDist = Sentences.Select(x => x.Label).Distinct().ToList();
labelDist.ForEach(label =>
{
// https://zhuanlan.zhihu.com/p/31197209
// calculate TF
// all words in the article
List<string> words = new List<string>();
Sentences.Where(x => x.Label == label).ToList().ForEach(sent =>
{
words.AddRange(sent.Words.Select(w => w.Text));
});
List<Tuple<string, double>> tfs = new List<Tuple<string, double>>();
words.Distinct().ToList().ForEach(w =>
{
// TF
int c1 = words.Count(x => x == w);
double tf = (c1 + 1.0) / words.Count();
// IDF
var sents = Sentences.Where(s => s.Words.Select(x => x.Text).Contains(w)).ToList();
double idf = Math.Log(Sentences.Count / (sents.Count() + 1.0));
tfs.Add(new Tuple<string, double>(w, tf * idf));
});
tfs = tfs.OrderByDescending(x => x.Item2).Take(words.Count / 10).ToList();
});
sentence.Words.ForEach(w =>
{
int index = Words.IndexOf(w.Text.ToLower());
});
}
public List<string> EncodeAll()
{
InitDictionary();
Sentences.ForEach(sent => Encode(sent));
//Parallel.ForEach(Sentences, sent => Encode(sent));
return Words;
}
private List<string> InitDictionary()
{
if (Words == null)
{
Words = new List<string>();
Sentences.ForEach(x =>
{
Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
});
Words = Words.Distinct().OrderBy(x => x).ToList();
}
return Words;
}
/// <summary>
/// Normalizes a TF*IDF array of vectors using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors">List<List<double>></param>
/// <returns>List<List<double>></returns>
public static List<List<double>> Normalize(List<List<double>> vectors)
{
// Normalize the vectors using L2-Norm.
List<List<double>> normalizedVectors = new List<List<double>>();
foreach (var vector in vectors)
{
var normalized = Normalize(vector);
normalizedVectors.Add(normalized);
}
return normalizedVectors;
}
/// <summary>
/// Normalizes a TF*IDF vector using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors"> List<double> </param>
/// <returns> List<double> </returns>
public static List<double> Normalize(List<double> vector)
{
List<double> result = new List<double>();
double sumSquared = 0;
foreach (var value in vector)
{
sumSquared += value * value;
}
double SqrtSumSquared = Math.Sqrt(sumSquared);
foreach (var value in vector)
{
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
result.Add(value / SqrtSumSquared);
}
return result;
}
}
}