service_mail/run.py

from __future__ import unicode_literals
#import imapclient
from config import Config
import sys
#from email.header import decode_header
#import email
import codecs

#import sys
#import bs4
import yaml
#sys.stdout = codecs.getwriter('utf8')(sys.stdout)
from storage.fetch_mail import fetch_mail
from storage.fetch_mail import fetch_threads, flatten_threads
from storage import Mail, MailThread, db_session
#import yaml
#import email
from classifier import get_training_threads,  print_answers,get_pipe, test_pipe, train_single_thread # , pipe2, pipe2b
from classifier import predict_threads
maintopic_values=["studium", "information","ausleihen","jobausschreibung", "umfragen"]

def predict_thread(p,l,t):
        pre=p.predict([t])
        print "Status is answered is estimated to be: " + str(l.inverse_transform(pre)[0])
        return pre
                

#print "arg1:"+sys.argv[1]
if len(sys.argv)>1:

    if sys.argv[1] == "fetch_threads":
        print flatten_threads(fetch_threads())
    if sys.argv[1] == "predict_threads":
        predict_threads()
    if sys.argv[1]=="stats":
            for topic in maintopic_values:
                    print topic
                    n_answ=db_session.query(MailThread).filter(MailThread.maintopic==topic).filter(MailThread.answered.op("IS")(True)).count()
                    n_nansw=db_session.query(MailThread).filter(MailThread.maintopic==topic).filter(MailThread.answered.op("IS NOT")(True)).count()
                    n_ges=db_session.query(MailThread).filter(MailThread.maintopic==topic).count()
                    print "%d answered and %d not answered of %d(%d) that are %d percent answerd" % (n_answ,n_nansw, n_ges,n_answ+n_nansw, float(n_answ)/float(n_ges)*100.0)



                                                                                
    if sys.argv[1] == "run_server":
            from flaskapp import app
            app.run(port=3000,debug=True)

    
    if sys.argv[1] == "train_thrd2":
        p, le=get_pipe("pipe2", "maintopic",["db"])
        pb, lb =get_pipe("pipe2b", "maintopic",["db"])
        
        train_single_thread(int(sys.argv[2]),p,le,b"maintopic")

    if sys.argv[1] == "train_thrd3":
#        p, le=get_pipe("pipe2", "maintopic")
        pb, lb =get_pipe("pipe2b", "lang")
        
        train_single_thread(int(sys.argv[2]),pb,lb,b"lang")

        
    if sys.argv[1] == "train_all2":
        p, labelencoder=train_fit_pipe2()
        pb, lb=train_fit_pipe2b()
        mth=db_session.query(MailThread).all()
        print mth
        for t in mth:
            if not in_training(t.firstmail,"maintopic"):
                print "---------------------------------------------------"
                print "---------------------------------------------------"
                print t.firstmail
                print t.text()
                predict_thread(pb,lb,t)
                train_single_thread(t.firstmail, p, labelencoder, b"maintopic") 
    if sys.argv[1] == "benchpipe3":
        test_pipe(["pipe2d","pipe2e","pipe2e1","pipe2f","pipe2g"],"maintopic",["db","de"])

    if sys.argv[1] == "benchpipe2":
        test_pipe(["pipe2","pipe2b","pipe2c","pipe2d"],"maintopic",["db","de"])
#        print "testing with db training data:"
#        test_pipe(["pipe2b"],"maintopic",["db"])
#        test_pipe(["pipe2b"],"maintopic",["db"])
#        print "testing only with german data"
#        test_pipe(["pipe2b"],"maintopic",["db","de"])

    if sys.argv[1] == "testpipe2":
        from classifier import ThreadSubjectExtractor, ThreadTextExtractor
        pipe2,le=train_fit_pipe2()
        
        if len(sys.argv)>2:
                t=db_session.query(MailThread).filter(MailThread.firstmail==sys.argv[2]).first()
                print t.to_text()
                print le.inverse_transform(pipe2.predict([t]))

    

    if sys.argv[1] == "train_thrd":
        pipe1, labelencoder=train_fit_pipe()
        train_single_thread(int(sys.argv[2]),pipe1,labelencoder)        

    if sys.argv[1] == "train_all":
        pipe1, labelencoder=train_fit_pipe()
        mth=db_session.query(MailThread).all()
        print mth
        for t in mth:
            if not in_training(t.firstmail):
                print "---------------------------------------------------"
                print "---------------------------------------------------"
                print t.firstmail
                train_single_thread(t.firstmail,pipe1,labelencoder)        
        
    if sys.argv[1] == "print_thread":
        mth=db_session.query(MailThread).filter(MailThread.firstmail==int(sys.argv[2])).first()
        print mth.mail_dicts()
        print mth.mail_flat_dict()
    
    if sys.argv[1] == "store_threads":
        thrds=flatten_threads(fetch_threads())
        for t in thrds:
            if type(t[0]) is int:
                th=db_session.query(MailThread).filter(MailThread.firstmail==t[0]).first()
                if th == None:
                    th=MailThread()
                    th.firstmail=t[0]
                if not th.body == yaml.dump(t):
                    th.body=yaml.dump(t)
                    th.islabeled=False
                    th.opened=True
                    th.compile()
                    db_session.add(th)
                    db_session.commit()
        print thrds
    
    if sys.argv[1] == "print_raw_mail":
        mm=db_session.query(Mail).filter(Mail.id==int(sys.argv[2])).first()
        print yaml.load(mm.envelope)

    if sys.argv[1] == "print_mail":
        mm=db_session.query(Mail).filter(Mail.id==int(sys.argv[2])).first()
        mm.compile_text()
        mm.compile_envelope()
        print mm.subject
        print "----------"
        print mm.text
    
    if sys.argv[1] == "mail_dict_test":
        mm=db_session.query(Mail).filter(Mail.id==int(sys.argv[2])).first()
        mm.compile_envelope()
        print mm.dict_envelope()

    
    if sys.argv[1] == "load_mail":
        mm=db_session.query(Mail).filter(Mail.id==int(sys.argv[2])).first()
        mm.compile_text()
        print mm.text
        env=yaml.load(mm.envelope)
        print env.subject
        print env

    
    if sys.argv[1] == "store_mail":
        m=fetch_mail(int(sys.argv[2]))
        mm=Mail()
        mm.envelope=yaml.dump(m['ENVELOPE'])
        mm.body=yaml.dump(m['RFC822'])
        mm.id=m['id']
        db_session.add(mm)
        db_session.commit()
    

   

    if sys.argv[1] == "initdb":
        from storage import init_db
        init_db()