bofh/service_mail
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rewrite classifier

Browse Source
This commit is contained in:
Andreas Stephanides
2017-08-07 10:20:28 +02:00
parent ff0bdc6d3b
commit 94d8d26187
11 changed files with 411 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
run.py
View File

@@ -1,87 +1,30 @@
from __future__ import unicode_literals
import imapclient
#import imapclient
from config import Config
import sys
from email.header import decode_header
import email
#from email.header import decode_header
#import email
import codecs
import sys
import bs4
#import sys
#import bs4
#sys.stdout = codecs.getwriter('utf8')(sys.stdout)
from storage.fetch_mail import fetch_mail
from storage.fetch_mail import fetch_threads, flatten_threads
from storage import Mail, MailThread, db_session
import yaml
import email
from classifier import get_training_threads, ThreadDictExtractor, print_answers, in_training, store_training_data, get_pipe, test_pipe # , pipe2, pipe2b
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
import numpy
#import yaml
#import email
from classifier import get_training_threads, print_answers, in_training, store_training_data, get_pipe, test_pipe, train_single_thread # , pipe2, pipe2b
from flaskapp import app
def train_fit_pipe():
tt= get_training_threads(b"answered")
# print tt[1]
# print tt[0]
pipe1.fit(tt[0],tt[1])
return pipe1,tt[2]
def train_fit_pipe2():
tt= get_training_threads(b"maintopic")
pipe2.fit(tt[0],tt[1])
return pipe2,tt[2]
def train_fit_pipe2b():
tt= get_training_threads(b"maintopic")
pipe2b.fit(tt[0],tt[1])
return pipe2b,tt[2]
def predict_thread(p,l,t):
pre=p.predict([t])
print "Status is answered is estimated to be: " + str(l.inverse_transform(pre)[0])
return pre
def train_single_thread(tid,p,le,key="answered"):
if (not type(tid) is int): raise TypeError("ID must be of type int")
if not type(p) is Pipeline: raise TypeError("Second Argument needs to be type Pipeline")
if not type(le) is LabelEncoder: raise TypeError("Second Argument needs to be type LabelEncoder")
mth=db_session.query(MailThread).filter(MailThread.firstmail==tid).first()
if mth is None: raise ValueError("Thread with firstmail %d not in Database" %tid)
# Predict the value
pre=p.predict([mth])
answ=pre[0]
#
# print mth.to_text()
# print mth.text()
print "Status is answered is estimated to be: " + str(le.inverse_transform(pre)[0])
print_answers(le)
ca=raw_input("Correct answer..")
try:
ca=int(ca)
except ValueError:
print "String Data"
if type(ca)==int:
if ca == answ:
print ("Yes I got it right")
else:
print("Oh no...!")
l=le.inverse_transform([ca])[0]
if type(l) is numpy.bool_:
l=bool(l)
if type(l) is numpy.string_:
l=str(l)
store_training_data(tid,l, key)
elif not ca.strip() == "":
store_training_data(tid, ca, key)
else:
print "couldn't handle %s" % ca
from flaskapp import app
#print "arg1:"+sys.argv[1]
if len(sys.argv)>1:
if sys.argv[1] == "fetch_threads":
@@ -89,6 +32,7 @@ if len(sys.argv)>1:
if sys.argv[1] == "run_server":
app.run(port=3000,debug=True)
if sys.argv[1] == "print_threads":
mth=db_session.query(MailThread).all()
for t in mth:
@@ -122,7 +66,15 @@ if len(sys.argv)>1:
p, le=get_pipe("pipe2", "maintopic")
pb, lb =get_pipe("pipe2b", "maintopic")
train_single_thread(int(sys.argv[2]),p,le,b"maintopic")
train_single_thread(int(sys.argv[2]),p,le,b"maintopic")
if sys.argv[1] == "train_thrd3":
# p, le=get_pipe("pipe2", "maintopic")
pb, lb =get_pipe("pipe2b", "lang")
train_single_thread(int(sys.argv[2]),pb,lb,b"lang")
if sys.argv[1] == "train_all2":
p, labelencoder=train_fit_pipe2()
pb, lb=train_fit_pipe2b()