151 lines
4.6 KiB
Python
151 lines
4.6 KiB
Python
from sqlalchemy import Column, Integer, String, Boolean, DateTime, Text, ForeignKey, Unicode
|
|
from sqlalchemy.orm import relationship
|
|
from datetime import datetime
|
|
from database import Base
|
|
from database import db_session
|
|
from email.header import decode_header
|
|
from marshmallow import Schema, fields, post_load
|
|
import yaml
|
|
import email
|
|
from fetch_mail import fetch_mail
|
|
import bs4
|
|
import re
|
|
class FullMailSchema(Schema):
|
|
id=fields.Integer()
|
|
text=fields.String()
|
|
body=fields.String()
|
|
envelope=fields.String()
|
|
|
|
|
|
|
|
class Mail(Base):
|
|
__tablename__ = 'mails'
|
|
id = Column(Integer, primary_key=True)
|
|
date = Column(DateTime)
|
|
envelope = Column(Text)
|
|
body = Column(Text)
|
|
text = Column(Text)
|
|
from_ = Column(Text)
|
|
from_mailbox=Column(String)
|
|
from_host=Column(String)
|
|
to_ = Column(Text)
|
|
to_mailbox = Column(Text)
|
|
to_host=Column(String)
|
|
subject = Column(Text)
|
|
__schema__=FullMailSchema
|
|
__jsonid__='mail'
|
|
__whiteattrs__= ["text", "envelope"]
|
|
__jsonattrs__=None
|
|
|
|
|
|
@classmethod
|
|
def fetch_mail(self,mid):
|
|
m=fetch_mail(mid)
|
|
mm=Mail()
|
|
mm.envelope=yaml.dump(m['ENVELOPE'])
|
|
em=email.message_from_string(m['RFC822'])
|
|
if type(em.get_payload()) is list:
|
|
pt=[]
|
|
for p in em.walk():
|
|
if p.get_content_maintype() == "text":
|
|
pt.append(p)
|
|
em.set_payload(pt)
|
|
mm.body=yaml.dump(str(em))
|
|
mm.id=m['id']
|
|
db_session.add(mm)
|
|
db_session.commit()
|
|
return mm
|
|
|
|
def get_email(self):
|
|
if self.body is None:
|
|
raise ValueError("body not yet loaded")
|
|
em=email.message_from_string(yaml.load(self.body))
|
|
return em
|
|
|
|
def compile_envelope(self):
|
|
env=yaml.load(self.envelope)
|
|
hd=decode_header(env.subject)
|
|
hd2=[]
|
|
|
|
for h in hd:
|
|
if not h[1] is None:
|
|
hd2.append(h[0].decode(h[1]))
|
|
# print h[0].decode(h[1])
|
|
else:
|
|
hd2.append(h[0])
|
|
self.subject=yaml.dump(hd2)
|
|
to_array=[]
|
|
from_array=[]
|
|
# print "Status"
|
|
# print env
|
|
if not env.date is None:
|
|
self.date=env.date
|
|
if not env.to is None:
|
|
for t in env.to:
|
|
a={"host": t.host, "mail": t.mailbox}
|
|
to_array.append(a)
|
|
self.to_=yaml.dump(to_array)
|
|
for t in env.from_:
|
|
a={"host": t.host, "mail": t.mailbox}
|
|
from_array.append(a)
|
|
self.to_=yaml.dump(to_array)
|
|
self.from_=yaml.dump(from_array)
|
|
return None
|
|
|
|
def dict_envelope(self):
|
|
d={}
|
|
if self.to_ is None:
|
|
self.compile_envelope()
|
|
if self.to_ is None:
|
|
raise ValueError("Self.to_ of mail not yet compiled")
|
|
i=0
|
|
for p in yaml.load(self.to_):
|
|
if p["host"] is not None:
|
|
d["to_host_"+str(i)]=p["host"]
|
|
if p["mail"] is not None:
|
|
d["to_mailbox_"+str(i)]=p["mail"]
|
|
i=i+1
|
|
i=0
|
|
for p in yaml.load(self.from_):
|
|
if p["host"] is not None:
|
|
d["from_host_"+str(i)]=p["host"]
|
|
if p["mail"] is not None:
|
|
d["from_mailbox_"+str(i)]=p["mail"]
|
|
i=i+1
|
|
return d
|
|
|
|
def compile_text(self):
|
|
for p in self.get_email().walk():
|
|
if p.get_content_maintype()=="text":
|
|
pl=p.get_payload(decode=True)
|
|
# print pl
|
|
# print p.get_content_type()
|
|
if p.get_content_subtype()=="html":
|
|
|
|
b4=bs4.BeautifulSoup(pl,"html.parser")
|
|
[s.extract() for s in b4('script')]
|
|
[s.extract() for s in b4('style')]
|
|
self.text= yaml.dump(b4.get_text())
|
|
else:
|
|
self.text =yaml.dump( pl)
|
|
def print_head(self):
|
|
fr=yaml.load(self.from_)
|
|
return "Gesendet von: "+str(fr[0]["mail"])+"@"+str(fr[0]["host"])+" am "+ str(self.date)
|
|
|
|
def print_text(self):
|
|
txt=""
|
|
fr=yaml.load(self.from_)
|
|
# txt= txt+ "Gesendet von: "+str(fr[0]["mail"])+"@"+str(fr[0]["host"])+" am "+ str(self.date) + "\n"
|
|
t=yaml.load(self.text)
|
|
if type(t) is unicode:
|
|
#txt=txt
|
|
txt=txt+t
|
|
else:
|
|
t=t.decode("ISO-8859-1")
|
|
txt=txt+t
|
|
txt=re.sub(r'\n\s*\n',r'\n',txt)
|
|
txt=re.sub(r'<!--.*-->',r'',txt,flags=re.MULTILINE|re.DOTALL)
|
|
txt=re.sub(r'\s*>+ .*\n',r'',txt)
|
|
|
|
return txt
|