rewrite classifier
This commit is contained in:
Andreas Stephanides
4
classifier/__init__.py
Normal file
4
classifier/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from classifier import in_training, print_answers
|
||||
from classifier import get_pipe, test_pipe, get_training_threads
|
||||
from training import train_single_thread
|
||||
from classifier import store_training_data
|
||||
191
classifier/classifier.py
Normal file
191
classifier/classifier.py
Normal file
@@ -0,0 +1,191 @@
|
||||
from sklearn.base import BaseEstimator, TransformerMixin
|
||||
from sklearn.feature_extraction import DictVectorizer
|
||||
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.pipeline import Pipeline, FeatureUnion
|
||||
from sklearn.naive_bayes import MultinomialNB
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
import numpy as np
|
||||
import yaml
|
||||
from storage import MailThread,db_session
|
||||
from sklearn.metrics import accuracy_score
|
||||
|
||||
|
||||
with open("data.yml", 'r') as stream:
|
||||
try:
|
||||
train=yaml.load(stream)
|
||||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
|
||||
data_types= { "answered": bool, "maintopic": str, "lang": str}
|
||||
|
||||
def set_train_data(i,d,key=b"answered"):
|
||||
global train
|
||||
#------------------------------------
|
||||
if not data_types.has_key(key):
|
||||
raise ValueError("Key "+str(key)+" unknown")
|
||||
if not train.has_key(i) or train[i] is None:
|
||||
train[i]={}
|
||||
if not type(d) is data_types[key]:
|
||||
raise TypeError("Data - %s - for key "% d +str(key)+" must be " +str(data_types[key])+ " but it is "+ str(type(d)))
|
||||
#------------------------------------
|
||||
train[i][key]=d
|
||||
|
||||
|
||||
def store_training_data(i, d,key=b"answered"):
|
||||
set_train_data(i,d,key)
|
||||
with open("data.yml","w") as file:
|
||||
file.write(yaml.dump(train,default_flow_style=True))
|
||||
file.close()
|
||||
|
||||
|
||||
# Lade Trainingsdaten fuer einen angegebenen key (Label/Eigenschaft)
|
||||
def get_training_threads(key="answered", filter=[]):
|
||||
if not data_types.has_key(key):
|
||||
raise ValueError("Key "+str(key)+" unknown")
|
||||
#------------------------------------
|
||||
t_a=[]
|
||||
d_a=[]
|
||||
d_a2=[]
|
||||
#------------------------------------
|
||||
for i in train:
|
||||
if train[i].has_key(key): # In den Trainingsdaten muss der relevante Key sein
|
||||
t=db_session.query(MailThread).filter(MailThread.firstmail==i).first()
|
||||
if not t is None: # Thread muss in der Datenbank sein
|
||||
t_a.append(t)
|
||||
d_a.append(train[i][key])
|
||||
le=LabelEncoder()
|
||||
d_a2=le.fit_transform(d_a)
|
||||
return (t_a,d_a2,le)
|
||||
|
||||
|
||||
def in_training(i, key="answered"):
|
||||
return train.has_key(i) and train[i].has_key(key)
|
||||
|
||||
|
||||
def print_answers(l):
|
||||
|
||||
cc=l.classes_
|
||||
c_id=l.transform(cc)
|
||||
for i,c in enumerate(cc):
|
||||
print str(i) + ": " + str(c)
|
||||
return None
|
||||
|
||||
|
||||
class ThreadDictExtractor(BaseEstimator, TransformerMixin):
|
||||
def fit(self, x, y=None):
|
||||
return self
|
||||
def transform(self, X,y=None):
|
||||
return [t.mail_flat_dict() for t in X]
|
||||
|
||||
class ThreadSubjectExtractor(BaseEstimator, TransformerMixin):
|
||||
def fit(self, x, y=None):
|
||||
return self
|
||||
def transform(self, X,y=None):
|
||||
return [t.subject() for t in X]
|
||||
|
||||
class ThreadTextExtractor(BaseEstimator, TransformerMixin):
|
||||
def fit(self, x, y=None):
|
||||
return self
|
||||
def transform(self, X,y=None):
|
||||
return [t.text() for t in X]
|
||||
|
||||
def get_pipe(p=b"pipe1",k=b"answered"):
|
||||
p=build_pipe(p)
|
||||
tt= get_training_threads(k)
|
||||
if len(tt[0]) > 0:
|
||||
p.fit(tt[0],tt[1])
|
||||
return p,tt[2]
|
||||
else:
|
||||
return None, None
|
||||
|
||||
def test_pipe(pp,k):
|
||||
tt= get_training_threads(k)
|
||||
X_train,X_test,y_train,y_test=train_test_split(tt[0],tt[1],test_size=0.2)
|
||||
if type(pp) is list:
|
||||
for p in pp:
|
||||
print "pipe: %s" % p
|
||||
p=build_pipe(p)
|
||||
p.fit(X_train,y_train)
|
||||
ypred=p.predict(X_test)
|
||||
print accuracy_score(y_test,ypred)
|
||||
|
||||
|
||||
|
||||
|
||||
def build_pipe(p=b"pipe1"):
|
||||
if p == "pipe1":
|
||||
p=Pipeline([('tde', ThreadDictExtractor()),
|
||||
('dv',DictVectorizer()),
|
||||
('clf', MultinomialNB())
|
||||
])
|
||||
elif p=="pipe2":
|
||||
p = Pipeline([
|
||||
('union', FeatureUnion(transformer_list=[
|
||||
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('text', Pipeline([('tte',ThreadTextExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('envelope', Pipeline([('tde', ThreadDictExtractor()),
|
||||
('dv',DictVectorizer())
|
||||
]))
|
||||
], transformer_weights={
|
||||
'subject': 1,
|
||||
'text': 0.7,
|
||||
'envelope': 0.7
|
||||
} )),
|
||||
('clf', MultinomialNB())
|
||||
])
|
||||
elif p=="pipe2b":
|
||||
p = Pipeline([
|
||||
('union', FeatureUnion(transformer_list=[
|
||||
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('text', Pipeline([('tte',ThreadTextExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('envelope', Pipeline([('tde', ThreadDictExtractor()),
|
||||
('dv',DictVectorizer())
|
||||
]))
|
||||
], transformer_weights={
|
||||
'subject': 1,
|
||||
'text': 0.7,
|
||||
'envelope': 0.7
|
||||
} )),
|
||||
('mlc', MLPClassifier())
|
||||
])
|
||||
elif p=="pipe2c":
|
||||
p = Pipeline([
|
||||
('union', FeatureUnion(transformer_list=[
|
||||
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('text', Pipeline([('tte',ThreadTextExtractor()),
|
||||
('cv',CountVectorizer()),
|
||||
('tfidf', TfidfTransformer())
|
||||
])),
|
||||
('envelope', Pipeline([('tde', ThreadDictExtractor()),
|
||||
('dv',DictVectorizer())
|
||||
]))
|
||||
], transformer_weights={
|
||||
'subject': 1,
|
||||
'text': 1,
|
||||
'envelope': 0.4
|
||||
} )),
|
||||
('mlc', MLPClassifier())
|
||||
])
|
||||
else:
|
||||
raise ValueError("The pipe %s is not a valid pipe")
|
||||
return p
|
||||
|
||||
|
||||
|
||||
25
classifier/classify_mail.py
Normal file
25
classifier/classify_mail.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from sklearn.feature_extraction.text import TfidfTransformer,CountVectorizer
|
||||
from sklearn.feature_extraction import DictVectorizer
|
||||
from sklearn.naive_bayes import MultinomialNB
|
||||
from sklearn.pipeline import Pipeline, FeatureUnion
|
||||
import sys
|
||||
import yaml
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
|
||||
|
||||
text_clf = Pipeline([('vect', CountVectorizer()),('tfidf', TfidfTransformer()),('clf', MultinomialNB())])
|
||||
|
||||
text_ohc = Pipeline([('ohc', OneHotEncoder()),('clf', MultinomialNB())])
|
||||
|
||||
combined_features = FeatureUnion([('vect1', CountVectorizer()),('vect2', CountVectorizer())])
|
||||
|
||||
|
||||
enc=OneHotEncoder()
|
||||
with open("example_1.yaml", 'r') as stream:
|
||||
try:
|
||||
train=yaml.safe_load(stream)
|
||||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
|
||||
tc=text_clf.fit(train["data"],train["target"])
|
||||
42
classifier/classify_text.py
Normal file
42
classifier/classify_text.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
|
||||
from sklearn.naive_bayes import MultinomialNB
|
||||
from sklearn.pipeline import Pipeline
|
||||
text_clf = Pipeline([('vect', CountVectorizer()),('tfidf', TfidfTransformer()),('clf', MultinomialNB())])
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
|
||||
|
||||
with open("example_1.yaml", 'r') as stream:
|
||||
try:
|
||||
train=yaml.safe_load(stream)
|
||||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
|
||||
tc=text_clf.fit(train["data"],train["target"])
|
||||
print(sys.argv[1])
|
||||
|
||||
answ=(tc.predict([sys.argv[1]]))[0]
|
||||
print train["target_names"][answ]
|
||||
|
||||
for i in range(0, (len(train["target_names"]))):
|
||||
print (str(i)+" "+ train["target_names"][i])
|
||||
|
||||
ca=int(raw_input("Correct answer.."))
|
||||
|
||||
|
||||
if ca == answ:
|
||||
print ("Yes I got it right")
|
||||
else:
|
||||
print("should I remember this?")
|
||||
a=raw_input("shoudIrememberthis?")
|
||||
if a == "y":
|
||||
train["data"].append(sys.argv[1])
|
||||
train["target"].append(ca)
|
||||
print yaml.dump(train,default_flow_style=False)
|
||||
file=open("example_1.yaml","w")
|
||||
file.write(yaml.dump(train,default_flow_style=False))
|
||||
file.close()
|
||||
else:
|
||||
print ("Ok, I already forgot")
|
||||
|
||||
70
classifier/training.py
Normal file
70
classifier/training.py
Normal file
@@ -0,0 +1,70 @@
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
import numpy
|
||||
from storage import Mail, MailThread, db_session
|
||||
from classifier import store_training_data, print_answers
|
||||
|
||||
|
||||
|
||||
def train_fit_pipe():
|
||||
tt= get_training_threads(b"answered")
|
||||
pipe1.fit(tt[0],tt[1])
|
||||
return pipe1,tt[2]
|
||||
def train_fit_pipe2():
|
||||
tt= get_training_threads(b"maintopic")
|
||||
pipe2.fit(tt[0],tt[1])
|
||||
return pipe2,tt[2]
|
||||
|
||||
def train_fit_pipe2b():
|
||||
tt= get_training_threads(b"maintopic")
|
||||
pipe2b.fit(tt[0],tt[1])
|
||||
return pipe2b,tt[2]
|
||||
|
||||
|
||||
def predict_thread(mth,p,le,key):
|
||||
#-------------------------------------------------------
|
||||
if not type(p) is Pipeline: raise TypeError("Second Argument needs to be type Pipeline")
|
||||
if not type(le) is LabelEncoder: raise TypeError("Second Argument needs to be type LabelEncoder")
|
||||
#-------------------------------------------------------
|
||||
pre=p.predict([mth])
|
||||
answ=pre[0]
|
||||
print "Status is answered is estimated to be: " + str(le.inverse_transform(pre)[0])
|
||||
return answ
|
||||
|
||||
def train_single_thread(tid,p,le,key="answered"):
|
||||
if (not type(tid) is int): raise TypeError("ID must be of type int")
|
||||
|
||||
mth=db_session.query(MailThread).filter(MailThread.firstmail==tid).first()
|
||||
if mth is None: raise ValueError("Thread with firstmail %d not in Database" %tid)
|
||||
print mth.firstmail
|
||||
print mth.subject()
|
||||
print mth.text()
|
||||
|
||||
if not p is None and not le is None:
|
||||
answ=predict_thread(mth,p,le,key)
|
||||
else: answ=None
|
||||
if not le is None:
|
||||
print_answers(le)
|
||||
|
||||
ca=raw_input("Correct answer..")
|
||||
try:
|
||||
ca=int(ca)
|
||||
except ValueError:
|
||||
print "String Data"
|
||||
|
||||
if type(ca)==int:
|
||||
if ca == answ:
|
||||
print ("Yes I got it right")
|
||||
else:
|
||||
print("Oh no...!")
|
||||
l=le.inverse_transform([ca])[0]
|
||||
if type(l) is numpy.bool_:
|
||||
l=bool(l)
|
||||
if type(l) is numpy.string_:
|
||||
l=str(l)
|
||||
store_training_data(tid,l, key)
|
||||
elif not ca.strip() == "":
|
||||
store_training_data(tid, ca, key)
|
||||
else:
|
||||
print "couldn't handle %s" % ca
|
||||
|
||||
Reference in New Issue
Block a user