init learning cats

classifier.py

@@ -0,0 +1,106 @@
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.feature_extraction import DictVectorizer
+from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.pipeline import Pipeline, FeatureUnion
+from sklearn.naive_bayes import MultinomialNB
+
+import numpy as np
+import yaml
+from storage import MailThread,db_session
+
+with open("data.yml", 'r') as stream:
+    try:
+        train=yaml.load(stream)
+    except yaml.YAMLError as exc:
+        print(exc)
+
+data_types= { "answered": bool, "maintopic": str}
+
+def store_training_data(i, d,key=b"answered"):
+    global train
+    if not data_types.has_key(key):
+        raise ValueError("Key "+str(key)+" unknown")
+    if not train.has_key(i):
+        train[i]={}
+    if not key is None and type(train[i]) is dict:
+        if not type(d)  is data_types[key]:
+#            print str(type(d)) + " vs " + str(data_types[key])
+            raise TypeError("Data - %s - for key "% d +str(key)+" must be " +str(data_types[key])+ " but it is "+ str(type(d)))
+        train[i][key]=d
+        
+    
+    with open("data.yml","w") as file:
+        file.write(yaml.dump(train,default_flow_style=True))
+        file.close()
+
+
+# Lade Trainingsdaten fuer einen angegebenen key (Label/Eigenschaft) 
+def get_training_threads(key="answered"):
+    t_a=[]
+    d_a=[]
+    d_a2=[]
+    for i in train:
+        t=db_session.query(MailThread).filter(MailThread.firstmail==i).first()
+        if not t is None:   # Thread muss in der Datenbank sein
+            if train[i].has_key(key): # In den Trainingsdaten muss der relevante Key sein
+                t_a.append(t)
+                d_a.append(train[i][key])
+    le=LabelEncoder()
+    d_a2=le.fit_transform(d_a)
+    return (t_a,d_a2,le)
+
+
+def in_training(i, key="answered"):
+    return train.has_key(i) and train[i].has_key(key)
+
+
+def print_answers(l):
+    cc=l.classes_
+    c_id=l.transform(cc)
+    for i,c in enumerate(cc):
+        print str(i) + ":  " + str(c)
+    return None
+
+
+class ThreadDictExtractor(BaseEstimator, TransformerMixin):
+    def fit(self, x, y=None):
+        return self
+    def transform(self, X,y=None):
+        return [t.mail_flat_dict() for t in X]
+
+class ThreadSubjectExtractor(BaseEstimator, TransformerMixin):
+    def fit(self, x, y=None):
+        return self
+    def transform(self, X,y=None):
+        return [t.subject() for t in X]
+
+class ThreadTextExtractor(BaseEstimator, TransformerMixin):
+    def fit(self, x, y=None):
+        return self
+    def transform(self, X,y=None):
+        return [t.text() for t in X]
+
+
+pipe1=Pipeline([('tde', ThreadDictExtractor()),('dv',DictVectorizer()),('clf', MultinomialNB())])
+
+pipe2 = Pipeline([
+    ('union', FeatureUnion(transformer_list=[
+        ('subject', Pipeline([('tse', ThreadSubjectExtractor()),
+                              ('cv',CountVectorizer()),
+                              ('tfidf', TfidfTransformer())
+        ])),
+        ('text',    Pipeline([('tte',ThreadTextExtractor()),
+                              ('cv',CountVectorizer()),
+                              ('tfidf', TfidfTransformer())
+        ])),
+        ('envelope', Pipeline([('tde', ThreadDictExtractor()),
+                               ('dv',DictVectorizer())
+        ]))
+    ], transformer_weights={
+        'subject': 1,
+        'text': 0.7,
+        'envelope': 0.5
+    } )),
+    ('clf', MultinomialNB())
+])