bofh/service_mail
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactor1

Browse Source
This commit is contained in:
Andreas Stephanides
2017-08-28 09:08:47 +02:00
parent 699f4f6546
commit 630b982502
14 changed files with 274 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
classifier/__init__.py
View File

@@ -1,6 +1,7 @@
from classifier import in_training, print_answers
from classifier import print_answers
from classifier import get_pipe, test_pipe, get_training_threads
from classifier import store_training_data
#from classifier import store_training_data
#in_training,
from training import train_single_thread

117
classifier/classifier.py
View File

@@ -8,88 +8,18 @@ from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
import numpy as np
import yaml
from storage import MailThread,db_session
from sklearn.metrics import accuracy_score, confusion_matrix
from collections import namedtuple
from storage import get_training_threads #MailThread,db_session
#with open("data.yml", 'r') as stream:
# try:
# train=yaml.load(stream)
# except yaml.YAMLError as exc:
# print(exc)
data_types= { "answered": bool, "maintopic": str, "lang": str}
def set_train_data(i,d,key=b"answered"):
global train
#------------------------------------
if not data_types.has_key(key):
raise ValueError("Key "+str(key)+" unknown")
if not train.has_key(i) or train[i] is None:
train[i]={}
if not type(d) is data_types[key]:
raise TypeError("Data - %s - for key "% d +str(key)+" must be " +str(data_types[key])+ " but it is "+ str(type(d)))
#------------------------------------
train[i][key]=d
def store_training_data(i, d,key=b"answered"):
set_train_data(i,d,key)
with open("data.yml","w") as file:
file.write(yaml.dump(train,default_flow_style=True))
file.close()
# Lade Trainingsdaten fuer einen angegebenen key (Label/Eigenschaft)
def get_training_threads(key="answered", filters=[]):
if not data_types.has_key(key):
raise ValueError("Key "+str(key)+" unknown")
#------------------------------------
t_a=[]
d_a=[]
d_a2=[]
#------------------------------------
if "db" in filters:
q=db_session.query(MailThread).filter(MailThread.istrained.is_(True))
if "de" in filters:
q=q.filter(MailThread.lang=="de")
elif "en" in filters:
q=q.filter(MailThread.lang=="en")
tt=q.all()
for t in tt:
t_a.append(t)
if key =="answered":
d_a.append(t.is_answered())
elif key=="maintopic":
d_a.append(t.maintopic)
elif key=="lang":
d_a.append(t.lang)
else:
raise ValueError("Database Filter now required")
le=LabelEncoder()
d_a2=le.fit_transform(d_a)
return (t_a,d_a2,le)
# else:
# for i in train:
# if train[i].has_key(key): # In den Trainingsdaten muss der relevante Key sein
# t=db_session.query(MailThread).filter(MailThread.firstmail==i).first#()
# if not t is None: # Thread muss in der Datenbank sein
# t_a.append(t)
# d_a.append(train[i][key])
def in_training(i, key="answered"):
return train.has_key(i) and train[i].has_key(key)
PredictTool=namedtuple("PredictTool",["Pipeline","LabelEncoder"])
def print_answers(l):
cc=l.classes_
c_id=l.transform(cc)
for i,c in enumerate(cc):
print str(i) + ": " + str(c)
classes=l.classes_
classes_encoded=l.transform(cc)
for i, c in zip(classes_encoded,classes):
print str(i) + ": " + str(c)
return None
@@ -117,15 +47,14 @@ class ThreadFirstTextExtractor(BaseEstimator, TransformerMixin):
def transform(self, X,y=None):
return [t.text("first") for t in X]
def get_pipe(p=b"pipe1",k=b"answered",filters=[]):
def get_pipe(p=b"pipe1",key=b"answered",filters=["db"]):
p=build_pipe(p)
tt= get_training_threads(k,filters)
#print tt
if len(tt[0]) > 0:
p.fit(tt[0],tt[1])
return p,tt[2]
threads, labels, labelenc= get_training_threads(key,filters)
if len(threads) > 0:
p.fit(threads,labels)
return PredictTool(p,labelenc)
else:
return None, None
return PredictTool(None, None)
def test_pipe(pp,k,f=[]):
tt= get_training_threads(k,f)
@@ -142,13 +71,13 @@ def test_pipe(pp,k,f=[]):
def build_pipe(p=b"pipe1"):
if p == "pipe1":
def build_pipe(pipe=b"pipe1"):
if pipe == "pipe1":
p=Pipeline([('tde', ThreadDictExtractor()),
('dv',DictVectorizer()),
('clf', MultinomialNB())
])
elif p=="pipe2":
elif pipe=="pipe2":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -169,7 +98,7 @@ def build_pipe(p=b"pipe1"):
} )),
('clf', MultinomialNB())
])
elif p=="pipe2b":
elif pipe=="pipe2b":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -190,7 +119,7 @@ def build_pipe(p=b"pipe1"):
} )),
('mlc', MLPClassifier())
])
elif p=="pipe2d":
elif pipe=="pipe2d":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -217,7 +146,7 @@ def build_pipe(p=b"pipe1"):
('mlc', MLPClassifier())
])
elif p=="pipe2e":
elif pipe=="pipe2e":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -243,7 +172,7 @@ def build_pipe(p=b"pipe1"):
} )),
('mlc', MLPClassifier(hidden_layer_sizes=(100,100)))
])
elif p=="pipe2e1":
elif pipe=="pipe2e1":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -269,7 +198,7 @@ def build_pipe(p=b"pipe1"):
} )),
('mlc', MLPClassifier(hidden_layer_sizes=(100,100,50)))
])
elif p=="pipe2f":
elif pipe=="pipe2f":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -295,7 +224,7 @@ def build_pipe(p=b"pipe1"):
} )),
('mlc', MLPClassifier(hidden_layer_sizes=(100,100)))
])
elif p=="pipe2g":
elif pipe=="pipe2g":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),
@@ -321,7 +250,7 @@ def build_pipe(p=b"pipe1"):
} )),
('mlc', MLPClassifier(hidden_layer_sizes=(100,100,100)))
])
elif p=="pipe2c":
elif pipe=="pipe2c":
p = Pipeline([
('union', FeatureUnion(transformer_list=[
('subject', Pipeline([('tse', ThreadSubjectExtractor()),

23
classifier/oldstuff.py Normal file
View File

@@ -0,0 +1,23 @@
def set_train_data(i,d,key=b"answered"):
global train
#------------------------------------
if not data_types.has_key(key):
raise ValueError("Key "+str(key)+" unknown")
if not train.has_key(i) or train[i] is None:
train[i]={}
if not type(d) is data_types[key]:
raise TypeError("Data - %s - for key "% d +str(key)+" must be " +str(data_types[key])+ " but it is "+ str(type(d)))
#------------------------------------
train[i][key]=d
def store_training_data(i, d,key=b"answered"):
set_train_data(i,d,key)
with open("data.yml","w") as file:
file.write(yaml.dump(train,default_flow_style=True))
file.close()
def in_training(i, key="answered"):
return train.has_key(i) and train[i].has_key(key)
data_types= { "answered": bool, "maintopic": str, "lang": str}

23
classifier/prediction.py
View File

@@ -1,22 +1,31 @@
from classifier import get_training_threads, print_answers, in_training, store_training_data, get_pipe
from classifier import get_pipe
from storage import db_session, MailThread
def predict_threads():
pipe1,le=get_pipe("pipe1",b"answered",["db"])
"""
Predicts the language, topic and if a thread is anwered and writes that to the database. This function doesn't have a return value.
"""
# Loading pipes for the prediction of each thread
pipe1,le=get_pipe("pipe1",key=b"answered",filter=["db"])
pipe2,le2=get_pipe("pipe2g", b"maintopic",["db"])
pipe3,le3=get_pipe("pipe2b", b"lang",["db"])
# Loading untrained MailThreads:
q=db_session.query(MailThread).filter(MailThread.istrained.op("IS NOT")(True))
mail_threads=q.all()
if len(mail_threads) ==0:
raise ValueError("no untrained threads found")
raise StandardError("no untrained threads found in database")
answered=le.inverse_transform(pipe1.predict(mail_threads))
maintopic=le2.inverse_transform(pipe2.predict(mail_threads))
lang=le3.inverse_transform(pipe3.predict(mail_threads))
for i, t in enumerate(mail_threads):
t.answered=bool(answered[i])
t.opened=bool(answered[i])
t.maintopic=str(maintopic[i])
t.lang=str(lang[i])
t.answered, t.opened, t.maintopic, t.lang = ( bool(answered[i]),
bool(answered[i]),
str(maintopic[i]),
str(lang[i])
)
db_session.add(t)
db_session.commit()

2
classifier/training.py
View File

@@ -2,7 +2,7 @@ from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
import numpy
from storage import Mail, MailThread, db_session
from classifier import store_training_data, print_answers
from classifier import print_answers

5
flaskapp/__init__.py
View File

@@ -49,13 +49,16 @@ def store_value(id,key,value):
mth.opened=bool(value)
if key=="maintopic" and value in maintopic_values:
mth.maintopic=str(value)
if key=="lang" and value in maintopic_values:
if key=="lang" and value in ["de", "en"]:
mth.lang=str(value)
if key =="trained":
value = value in ["true", "True", "1", "t"]
mth.istrained=bool(value)
db_session.add(mth)
db_session.commit()
@app.route("/<int:id>")
def store_answered(id):

89
flaskapp/templates/_macros.html Normal file
View File

@@ -0,0 +1,89 @@
{# -*-jinja2-*- #}
{% macro render_nav(m, maintopics) %}
<div class="nav">
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink1_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#">
{% if m.is_answered() %}
IS answered
{% else %}
ISNOT answered
{% endif %}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink1_{{m.firstmail}}">
<a class="btn" href="{{m.firstmail}}?key=answered&value={{(not m.is_answered())}}">
answered:{{(not m.is_answered())}}
</a>
</div>
</div>
<div class="dropdown nav-item">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink2_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#">
{{m.maintopic}}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink2_{{m.firstmail}}">
{% for t in maintopics %}
<a class="dropdown-item" href="{{m.firstmail}}?key=maintopic&value={{t}}">{{t}}</a>
{% endfor %}
</div>
</div>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink3_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {% if m.istrained %}
IS trained
{% else %}
ISNOT trained
{% endif %}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink3_{{m.firstmail}}">
<a class="btn" href="{{m.firstmail}}?key=trained&value={{(not m.istrained)}}">TRAINED:{{(not m.istrained)}}</a>
</div>
</div>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink4_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {{ m.lang }}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink3_{{m.firstmail}}">
{% for l in ["de", "en"] %}
<a class="btn" href="{{m.firstmail}}?key=lang&value={{l}}">{{(l)}}</a>
{% endfor %}
</div>
</div>
</div>
{% endmacro %}
{% macro render_inline_thread(m, maintopics) %}
<div class="card card-thread answ-{{m.is_answered()}} trained-{{m.istrained}}">
<div class="" role="tab" id="heading{{m.firstmail}}">
<b class="mb-0">
<a data-toggle="collapse"
data-parent="#accordion"
href="#collapse{{m.firstmail}}"
aria-expanded="true" aria-controls="collapse1"
>
{% if m.istrained %} trained: {% endif %} {{m.tstr()}}
</a>
</b>
</div>
<div id="collapse{{m.firstmail}}"
class="collapse {{'show' if m.firstmail==opened}}"
role="tabpanel"
aria-labelledby="headingOne"
>
<div class="card-block">
{{render_nav(m,maintopics) }}
</div>
{% for (h,txt) in m.print_mail() %}
<div> {{h}}</div>
<div class="textbox"> {{ txt }} </div>
{% endfor %}
</div>
</div>
{% endmacro %}

133
flaskapp/templates/index.html
View File

@@ -1,104 +1,45 @@
{# -*-jinja2-*- #}
{% from "_macros.html" import render_inline_thread %}
{% extends "layout.html" %}
{% block header %}
<style>
.card.answ-1,.card.answ-True {
background: lightgreen;
}
.card.trained-1,.card.trained-True {
border: solid blue;
}
.card.card-thread {
padding-top: 2pt;
padding-bottom:2pt;
border-radius:0;
margin-top:1pt;
margin-bottom:1pt
}
div.textbox {
white-space: pre-wrap;
font:Courier, monospace;
font-size:small;
width:50em;
border: thin blue solid;
}
</style>
{% endblock %}
<html>
<head>
<title>{{title}}</title>
<script src="/static/jquery-3.2.0.min.js" ></script>
<link rel="stylesheet" href="/static/bootstrap/css/bootstrap.min.css"/>
<script src="/static/bootstrap/js/bootstrap.min.js" ></script>
</head>
<body>
<style>
.card.answ-1,.card.answ-True {
background: lightgreen;
}
.card.trained-1,.card.trained-True {
border: solid blue;
}
</style>
<div class="container">
<div class="row">
<div class="col-12">
<h1>{{title}}</h1>
<div id="accordion" role="tablist" aria-multiselectable="true">
{% for m in mths %}
<div class="card answ-{{m.is_answered()}} trained-{{m.istrained}}" style="padding-top: 2pt; padding-bottom:2pt; border-radius:0;margin-top:1pt; margin-bottom:1pt">
<div class="" role="tab" id="heading{{m.firstmail}}">
<b class="mb-0">
<a data-toggle="collapse" data-parent="#accordion" href="#collapse{{m.firstmail}}" aria-expanded="true" aria-controls="collapse1">
{% if m.istrained %} trained: {% endif %} {{m.tstr()}}
</a>
</b>
</div>
{% block content %}
<div class="container">
<div class="row">
<div class="col-12">
<div id="collapse{{m.firstmail}}" class="collapse {{'show' if m.firstmail==opened}}" role="tabpanel" aria-labelledby="headingOne">
<div class="card-block">
<div class="nav">
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink1_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {% if m.is_answered() %}
IS answered
{% else %}
ISNOT answered
{% endif %}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink1_{{m.firstmail}}">
<h1>{{title}}</h1>
<a class="btn" href="{{m.firstmail}}?key=answered&value={{(not m.is_answered())}}">answered:{{(not m.is_answered())}}</a>
</div>
</div>
<div class="dropdown nav-item">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink2_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {{m.maintopic}}</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink2_{{m.firstmail}}">
{% for t in maintopics %}
<a class="dropdown-item" href="{{m.firstmail}}?key=maintopic&value={{t}}">{{t}}</a>
{% endfor %}
</div>
</div>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink3_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {% if m.istrained %}
IS trained
{% else %}
ISNOT trained
{% endif %}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink3_{{m.firstmail}}">
<a class="btn" href="{{m.firstmail}}?key=trained&value={{(not m.istrained)}}">TRAINED:{{(not m.istrained)}}</a>
</div>
</div>
<div class="nav-item dropdown">
<a class="nav-link dropdown-toggle" id="dropdownMenuLink4_{{m.firstmail}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" href="#"> {{ m.lang }}
</a>
<div class="dropdown-menu" arialabelledby="dropdownMenuLink3_{{m.firstmail}}">
{% for l in ["de", "en"] %}
<a class="btn" href="{{m.firstmail}}?key=lang&value={{l}}">{{(l)}}</a>
{% endfor %}
</div>
</div>
</div>
</div>
{% for txt in m.print_text() %}
<div style="white-space: pre-wrap;font:Courier, monospace; font-size:small; width:50em; border: thin blue solid;">
{{ txt }}
</div>
{% endfor %}
</div>
</div>
<div id="accordion" role="tablist" aria-multiselectable="true">
{% for m in mths %}
{{ render_inline_thread(m, maintopics) }}
{% endfor %}
</div>
</div>
</div>
</div>
</div>
</body>
</div>
{% endblock %}

17
flaskapp/templates/layout.html Normal file
View File

@@ -0,0 +1,17 @@
{# -*-jinja2-*- #}
<html>
<head>
<title>{{title}}</title>
<script src="/static/jquery-3.2.0.min.js" ></script>
<link rel="stylesheet" href="/static/bootstrap/css/bootstrap.min.css"/>
<script src="/static/bootstrap/js/bootstrap.min.js" ></script>
{% block header %}
{% endblock %}
</head>
<body>
{% block content %}
{% endblock %}
</body>
</html>

30
run.py
View File

@@ -70,29 +70,6 @@ if len(sys.argv)>1:
from flaskapp import app
app.run(port=3000,debug=True)
if sys.argv[1] == "print_threads":
mth=db_session.query(MailThread).all()
for t in mth:
print t.firstmail
print t.mail_flat_dict()
if sys.argv[1] == "print_thrd":
if len(sys.argv)<3:
mth=db_session.query(MailThread).all()
for t in mth:
print t.firstmail
else:
t=db_session.query(MailThread).filter(MailThread.firstmail==sys.argv[2]).first()
print t.firstmail
print t.subject()
print t.text()
if sys.argv[1] == "compile_threads":
mth=db_session.query(MailThread).all()
l=len(mth)
for i,t in enumerate(mth):
print "%d/%d" % (i,l)
t.compile()
if sys.argv[1] == "trained_threads_from_yml":
from classifier.classifier import train
@@ -188,10 +165,9 @@ if len(sys.argv)>1:
th.body=yaml.dump(t)
th.islabeled=False
th.opened=True
else:
th.body=yaml.dump(t)
db_session.add(th)
db_session.commit()
th.compile()
db_session.add(th)
db_session.commit()
print thrds
if sys.argv[1] == "print_raw_mail":

36
storage/__init__.py
View File

@@ -2,7 +2,41 @@
from database import db_session, init_db
from database import Base
metadata=Base.metadata
from sklearn.preprocessing import LabelEncoder
from collections import namedtuple
# Two main objects: Mail & MailThread
from mail_model import Mail
from thread_model import MailThread
def str_bool(s):
return s in ["1", "true", "True", "t","T"]
# Lade Trainingsdaten fuer einen angegebenen key (Label/Eigenschaft)
def get_training_threads(key="answered", filters=[]):
#------------------------------------
db_fields= {"answered": lambda t: t.is_answered(),
"lang": lambda t: t.lang,
"maintopic": lambda t: t.maintopic}
if not db_fields.has_key(key):
raise ValueError("Key "+str(key)+" unknown")
q=db_session.query(MailThread)
q=q.filter(MailThread.istrained.is_(True))
if "de" in filters:
q=q.filter(MailThread.lang=="de")
elif "en" in filters:
q=q.filter(MailThread.lang=="en")
# load and extract thread fields
threads=q.all()
labels = map(db_fields[key], threads)
# encode using LabelEncoder
le=LabelEncoder()
labels=le.fit_transform(labels)
TrainingThreads=namedtuple("TrainingThreads", ["MailThreads","EncodedLabels","LabelEncoder"])
return TrainingThreads(threads,labels,le)

6
storage/mail_model.py
View File

@@ -128,10 +128,14 @@ class Mail(Base):
self.text= yaml.dump(b4.get_text())
else:
self.text =yaml.dump( pl)
def print_head(self):
fr=yaml.load(self.from_)
return "Gesendet von: "+str(fr[0]["mail"])+"@"+str(fr[0]["host"])+" am "+ str(self.date)
def print_text(self):
txt=""
fr=yaml.load(self.from_)
txt= txt+ "Gesendet von: "+str(fr[0]["mail"])+"@"+str(fr[0]["host"])+" am "+ str(self.date) + "\n"
# txt= txt+ "Gesendet von: "+str(fr[0]["mail"])+"@"+str(fr[0]["host"])+" am "+ str(self.date) + "\n"
t=yaml.load(self.text)
if type(t) is unicode:
#txt=txt

18
storage/thread_model.py
View File

@@ -105,6 +105,14 @@ class MailThread(Base):
db_session.commit()
self.date=self.mails()[0].date
def print_mail(self, filter="all"):
a=[]
if filter=="all":
mm=self.mails()
for m in mm:
a.append((m.print_head(), m.print_text()))
return a
def print_text(self,filter="all"):
a=[]
if filter=="all":
@@ -114,6 +122,16 @@ class MailThread(Base):
elif filter=="first":
a.append(m[0].print_text())
return a
def print_head(self,filter="all"):
a=[]
if filter=="all":
mm=self.mails()
for m in mm:
a.append(m.print_head())
elif filter=="first":
a.append(m[0].print_head())
return a
def text(self,filter="all"):
a=u""
def mail_txt(m):

BIN
test.sqlite
View File

Binary file not shown.