init learning cats

2017-08-04 07:49:39 +02:00
commit 941cbc3d45
14 changed files with 847 additions and 0 deletions
--- a/storage/mail_model.py
+++ b/storage/mail_model.py
@@ -0,0 +1,126 @@
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Text, ForeignKey, Unicode
+from sqlalchemy.orm import relationship
+from datetime import datetime
+from database import Base
+from database import db_session
+from email.header import decode_header
+from marshmallow import Schema, fields, post_load
+import yaml
+import email
+from fetch_mail import fetch_mail
+import bs4 
+class FullMailSchema(Schema):
+    id=fields.Integer()
+    text=fields.String()
+    body=fields.String()
+    envelope=fields.String()
+    
+
+
+class Mail(Base):
+    __tablename__ = 'mails'
+    id = Column(Integer, primary_key=True)
+    date = Column(DateTime)
+    envelope = Column(Text)
+    body = Column(Text)
+    text = Column(Text)
+    from_ = Column(Text)
+    from_mailbox=Column(String)
+    from_host=Column(String)
+    to_ = Column(Text)
+    to_mailbox = Column(Text)
+    to_host=Column(String)
+    subject = Column(Text)
+    __schema__=FullMailSchema
+    __jsonid__='mail'
+    __whiteattrs__= ["text", "envelope"]
+    __jsonattrs__=None
+
+
+    @classmethod
+    def fetch_mail(self,mid):
+        m=fetch_mail(mid)
+        mm=Mail()
+        mm.envelope=yaml.dump(m['ENVELOPE'])
+        em=email.message_from_string(m['RFC822'])
+        if type(em.get_payload()) is list:
+            pt=[]
+            for p in em.walk():
+                if p.get_content_maintype() == "text":
+                    pt.append(p)
+            em.set_payload(pt)
+        mm.body=yaml.dump(str(em))
+        mm.id=m['id']
+        db_session.add(mm)
+        db_session.commit()
+        return mm
+        
+    def get_email(self):
+        em=email.message_from_string(yaml.load(self.body))
+        return em    
+
+    def compile_envelope(self):
+        env=yaml.load(self.envelope)
+        hd=decode_header(env.subject)
+        hd2=[]
+
+        for h in hd:
+            if not h[1] is None:
+                hd2.append(h[0].decode(h[1]))
+                #            print h[0].decode(h[1])
+            else:
+                hd2.append(h[0])
+        self.subject=yaml.dump(hd2)
+        to_array=[]
+        from_array=[]
+#        print "Status"
+        #        print env
+        if env.to is  None:
+            print self.id
+        else:
+            for t in env.to:
+                a={"host": t.host, "mail": t.mailbox}
+                to_array.append(a)    
+        self.to_=yaml.dump(to_array)
+        for t in env.from_:
+            a={"host": t.host, "mail": t.mailbox}
+            from_array.append(a)    
+        self.to_=yaml.dump(to_array)
+        self.from_=yaml.dump(from_array)
+        return None
+
+    def dict_envelope(self):
+        d={}
+        i=0
+        for p in yaml.load(self.subject):
+            if p is not None:
+                d["subject_"+str(i)]=p
+            i=i+1
+        i=0
+        for p in yaml.load(self.to_):
+            if p["host"] is not None:
+                d["to_host_"+str(i)]=p["host"]
+            if p["mail"] is not None:
+                d["to_mailbox_"+str(i)]=p["mail"]
+            i=i+1
+        i=0
+        for p in yaml.load(self.from_):
+            if p["host"] is not None:
+                d["from_host_"+str(i)]=p["host"]
+            if p["mail"] is not None:
+                d["from_mailbox_"+str(i)]=p["mail"]
+            i=i+1    
+        return d
+    
+    def compile_text(self):
+        for p in self.get_email().walk():
+            if p.get_content_maintype()=="text":
+                pl=p.get_payload(decode=True)
+#                print pl
+#                print p.get_content_type()
+                if p.get_content_subtype()=="html":
+
+                    b4=bs4.BeautifulSoup(pl,"html.parser")
+                    self.text= yaml.dump(b4.get_text())
+                else:
+                    self.text =yaml.dump( pl)