organizations&organizationtypes

__init__.py

@@ -85,7 +85,8 @@ def after_request(response):
 from .articles.views import article_pages
 from .sections.views import section_pages
 from .compiler.views import compiler_pages
-
+from .organizations.views import organization_pages
+from .organizationtypes.views import organizationtype_pages
 
 @app.route("/")
 @app.route("/index")
@@ -97,6 +98,9 @@ def home():
 app.register_blueprint(article_pages, url_prefix='/articles')
 app.register_blueprint(section_pages, url_prefix='/sections')
 app.register_blueprint(compiler_pages, url_prefix='/compiler')
+app.register_blueprint(organization_pages, url_prefix='/organizations')
+app.register_blueprint(organization_pages, url_prefix='/organizations')
+app.register_blueprint(organizationtype_pages, url_prefix='/organizationtypes')
 
 
 # ------------ Telegram Bot

articles/controller.py

@@ -4,47 +4,54 @@ from .model import ArticleSchema
 from datetime import datetime
 import json
 from src.database import db_session, read_json
-
+from src.controller import BaseController
-
+class ArtController(BaseController):
-def pagination_params(v):
+    __myclass__=Article
-    try:
+    __jsonid__='article'
-        if v.has_key("per_page"):
+    def pagination_params(self,v):
-            pp=int(v["per_page"])
+        try:
-        else:
+            if v.has_key("per_page"):
+                pp=int(v["per_page"])
+            else:
+                pp=20
+            if v.has_key("page"):
+                o=(int(v["page"])-1) *pp
+            else:
+                o=0
+        except ValueError:
             pp=20
-        if v.has_key("page"):
-            o=(int(v["page"])-1) *pp
-        else:
             o=0
-    except ValueError:
+        if not (isinstance(pp,int) and pp>0 and pp<10000):
-        pp=20
+            pp=20
-        o=0
+        if not (isinstance(o,int) and o>=0 and o<100000):
-    if not (isinstance(pp,int) and pp>0 and pp<10000):
+            o=0
-        pp=20
+        p1=(pp,o)
-    if not (isinstance(o,int) and o>=0 and o<100000):
+        return p1
-        o=0
+
-    return (pp, o)
+    def search(self,s):
+        return Article.query.filter(Article.title.like("%"+s+"%")).order_by(Article.published_date.desc()).limit(20).all()
 
 
-def get_all():
+    def get_all(self):
-    return Article.query.order_by(Article.published_date.desc()).all()
+        return Article.query.order_by(Article.published_date.desc()).all()
-
-def search(s):
-    return Article.query.filter(Article.title.like("%"+s+"%")).order_by(Article.published_date.desc()).limit(20).all()
-
-def get_all_page(lim, off):
-    return Article.query.order_by(Article.published_date.desc()).limit(lim).offset(off).all()
 
 
-def get_section_page(section_id, lim, off):
+    def get_all_page(self,lim, off):
-    return Article.query.filter(Article.section_id==section_id).order_by(Article.published_date.desc()).limit(lim).offset(off).all()
+        return Article.query.order_by(Article.published_date.desc()).limit(lim).offset(off).all()
 
 
-def get_section_page(section_id, lim, off):
+    def get_section_page(self,section_id, lim, off):
-    return Article.query.filter(Article.section_id==section_id).order_by(Article.published_date.desc()).limit(lim).offset(off).all()
+        return Article.query.filter(Article.section_id==section_id).order_by(Article.published_date.desc()).limit(lim).offset(off).all()
 
-def section_count(section_id):
-    return Article.query.filter(Article.section_id==section_id).count()
 
-def count():
+    def get_section_page(section_id, lim, off):
-    return Article.query.count()
+        return Article.query.filter(Article.section_id==section_id).order_by(Article.published_date.desc()).limit(lim).offset(off).all()
+
+    def section_count(section_id):
+        return Article.query.filter(Article.section_id==section_id).count()
+
+    def count():
+        return Article.query.count()
+
+
+controller=ArtController()

articles/views.py

@@ -6,8 +6,6 @@ from .model import ArticleSchema
 from datetime import datetime
 import json
 
-#flask.json.JSONEncoder.default = lambda self,obj: (obj.isoformat() if isinstance(obj, datetime) else None)
-#flask.json.JSONEncoder.default = lambda self,obj: ((obj.dict()) if isinstance(obj, Article) else None)
 from src import clogger
 import json
 from src.database import db_session, read_json
@@ -15,7 +13,7 @@ import flask
 
 #flask.json.JSONEncoder.default = lambda self,obj: ((ArticleSchema().dump(obj)[0]) if isinstance(obj, Article) else None)
 flask.json.JSONEncoder.default = lambda self,obj: ((obj.__json__()) if isinstance(obj, (Base, Article,CrawlUrl)) else None) 
-import controller 
+from controller import controller 
 @article_pages.route("/")
 @article_pages.route("")
 @article_pages.route(".json")
@@ -57,7 +55,7 @@ def get(id):
 @article_pages.route("/<int:id>.json",methods=['DELETE'])
 def delete(id):
     article=Article.query.get(id)
-    clogger.info(id)
+#    clogger.info(id)
     if article != None:
         db_session.delete(article)
         db_session.commit()

controller.py

@@ -0,0 +1,47 @@
+#from src.models import Organization
+from flask import request
+from src.database import read_json, db_session
+from sqlalchemy.exc import IntegrityError
+from src import clogger
+
+class BaseController():
+    def get(self,id):
+        return self.__myclass__.query.get(id)
+
+    def create(self,request):
+        d=read_json(request)
+        clogger.info(d)
+        clogger.info(d[self.__jsonid__])
+        o=self.__myclass__()
+        success, errors=o.update(d[self.__jsonid__])
+        if success:
+            db_session.add(o)
+            try:
+                db_session.commit()
+            except IntegrityError as e:
+                db_session.rollback()
+                errors.append(e.message)        
+        return o, errors
+
+    def get_all(self):
+        return self.__myclass__.query.all()
+    
+    def delete(self,id):
+        o=self.get(id)
+        if o != None:
+            db_session.delete(o)
+            db_session.commit()
+        
+    def update(self,id,request):
+        a=read_json(request)
+        o=self.get(id)
+        success,errors=o.update(a[self.__jsonid__])
+        if success:
+            try:
+                db_session.commit()
+            except IntegrityError as e:
+                db_session.rollback()
+                errors.append(e.message)        
+
+        return o,errors
+

database.py

@@ -36,9 +36,14 @@ engine2 = create_engine('sqlite:///'+  path.join(db_path,db_urlfile), convert_un
 db_session2 = scoped_session(sessionmaker(autocommit=False,
                                          autoflush=False,
                                           bind=engine2))
+from database_mbase import MyBase
 
-Base = declarative_base()
+
+#Base = declarative_base()
+#Base.query = db_session.query_property()
+Base=declarative_base(cls=MyBase)
 Base.query = db_session.query_property()
+
 Base2 = declarative_base()
 Base2.query = db_session2.query_property()
 
@@ -48,11 +53,14 @@ def read_json(rq):
     if js is None:
         js=rq.form.to_dict()
     if js=={} and rq.data != "":
-        js=json.loads(rq.data)    
+        d=rq.data
+        js=json.loads(d)
+    clogger.info(js)
     return js
 
 def init_db():
     import src.models
+    from src.models import Organization
     Base.metadata.create_all(bind=engine)
 
 def init_db2():

database_mbase.py

@@ -0,0 +1,35 @@
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Text, ForeignKey, Index, TIMESTAMP
+from src import clogger
+from datetime import datetime
+class MyBase(object):
+    id = Column(Integer, primary_key=True)
+    created_at = Column(TIMESTAMP, default=datetime.utcnow, nullable=False)
+    updated_at = Column(TIMESTAMP, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
+    def __json__(self):
+        if self.__jsonattrs__ is None:
+            return self.__schema__().dump(self)[0]
+        else:
+            return self.__schema__(only=self.__jsonattrs__).dump(self)[0]
+#    def __init__(self, data={}):
+#        self.update(data,False)
+        
+    def update(self,data, partial=True):
+        data, errors=self.__schema__( only=self.__whiteattrs__).load(data, partial=partial)
+        if len(errors)>0:
+            clogger.error(errors)
+            return (False,errors)
+        else:
+            for a in self.__whiteattrs__:
+                if data.has_key(a):
+                    setattr(self,a,data[a])
+            return (True, [])
+
+    @classmethod
+    def deserialize(cls,data):
+        data, errors=cls.__schema__().load(data,partial=True)
+        a=cls()
+        for c in cls.__table__.columns:
+            if data.has_key(c.key):
+                setattr(a, c.key,data[c.key])
+        return a
+

init_database.py

@@ -0,0 +1,3 @@
+from src.database import init_db
+
+init_db()

load_articles.py

@@ -1,6 +1,7 @@
 import sys
 import json
 from src.articles.model import Article, FullArticleSchema
+from src.sections.model import Section, FullSectionSchema
 from src.database import db_session
 from sqlalchemy.exc import IntegrityError
 
@@ -9,10 +10,10 @@ if len(sys.argv) <= 1:
 
 
 def load_article(a):
-    return FullArticleSchema().load(a[0]).data
+    return Article.deserialize(a[0])
 
 def load_section(s):
-    return FullSectionSchema().load(s[0]).data
+    return Section.deserialize(s[0])
 
 
 
@@ -23,7 +24,7 @@ articles=None
 sections=None
 organizations=None
 
-if isinstace(data,dict):
+if isinstance(data,dict):
     if data.has_key("articles"):
         articles=data["articles"]
     if data.has_key("sections"):
@@ -34,7 +35,8 @@ else:
 
 
 articles= map (load_article, articles)
-sections=map(load_section,sections)
+if sections is not None:
+    sections=map(load_section, sections)
 
 for a in articles:
     try:
@@ -44,14 +46,15 @@ for a in articles:
         db_session.rollback()
     finally:
         db_session.rollback()
+if sections is not None:
+    for s in sections:
+        if not isinstance(s,Section):
+            print type(s)
+        try:
+            db_session.add(s)
+            db_session.commit()
+        except IntegrityError:
+            db_session.rollback()
 
-for s in sections:
-    try:
-        db_session.add(s)
-        db_session.commit()
-    except IntegrityError:
-        db_session.rollback()
-    finally:
-        db_session.rollback()
         
 file.close()

models.py

@@ -1,4 +1,8 @@
 
 from .articles.model import Article
 from .sections.model import Section
+from .organizations.model import Organization
+from .organizationtypes.model import Organizationtype
+
 from .compiler.models import CrawlUrl, CrawlCache
+

organizations/controller.py

@@ -0,0 +1,21 @@
+from src.controller import BaseController
+from model import Organization
+from src.models import Article, Section
+from src.database import db_session
+class OrgController(BaseController):
+    __myclass__= Organization
+    __jsonid__ = 'organization'
+    def get_articles(self,id):
+        sid=db_session.query(Section.id).filter(Section.organization_id==id).all()
+        sid=map(lambda a:a[0], sid)
+        articles=Article.query.filter(Article.section_id.in_(sid)).all()
+        return articles
+
+    def get_by_key_articles(self,key):
+        org=self.get_by_key(key)
+        return self.get_articles(org.id)
+    
+    def get_by_key(self,key):
+        return Organization.query.filter(Organization.key==key).one()
+    
+controller=OrgController()

organizations/model.py

@@ -0,0 +1,65 @@
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Text, ForeignKey, Index, TIMESTAMP
+from sqlalchemy.orm import relationship, validates
+from src.organizationtypes.model import Organizationtype
+from datetime import datetime
+from src.database import Base,db_session
+from marshmallow import Schema, fields, post_load, ValidationError
+from src import clogger
+import json
+import flask 
+from src.models import Section
+import re
+
+def validate_key(key):
+    if re.search('[\s)(?*/&+?]+', key) is not None:
+        raise ValidationError("Special Character not allowed")
+    if key.strip()=="":
+        raise ValidationError("No empty keys allowed")
+    return key
+
+class OrganizationtypeCompSchema(Schema):
+    id =fields.Integer()
+    name=fields.String()
+
+class FullOrganizationSchema(Schema):
+    id = fields.Integer()
+    key=fields.String(validate=validate_key)
+    url = fields.String()
+    name = fields.String()
+    image = fields.String()
+    updated_at = fields.DateTime()
+    created_at = fields.DateTime()
+#    organizationtype_id = fields.Integer()
+    organizationtype=fields.Nested(OrganizationtypeCompSchema)
+
+class Organization(Base):
+    __tablename__ = 'organizations'
+    url = Column(String(250))
+    name=Column(String(250),nullable=False)
+    key=Column(String(250), unique=True, nullable=False)
+    Index("key", "key")
+    sections=relationship("Section", back_populates="organization")
+    image=Column(String(250))
+    organizationtype_id=Column(Integer, ForeignKey('organizationtypes.id'))
+    organizationtype=relationship("Organizationtype")
+    text = Column(Text)    
+    __schema__= FullOrganizationSchema
+    __jsonid__= 'organization'
+    __whiteattrs__= ["url", "name", "image","key", "organizationtype_id"]
+    __jsonattrs__= ["id", "url", "name", "updated_at", "created_at", "image", "key","organizationtype.name","text" ] # None means all
+
+    @validates('key')
+    def _validate_key(self,k, key):
+        return validate_key(key)
+    def __init__(self, data={}):
+        self.update(data,False)
+
+
+    
+        
+
+#class OrganizationSchema(Schema):
+#    id = fields.Integer()
+#    url = fields.String()
+#    name = fields.String()
+#    image = fields.String()

organizations/views.py

@@ -0,0 +1,73 @@
+from flask import Blueprint, jsonify, render_template, abort, redirect, url_for, request
+
+organization_pages = Blueprint('organizations', __name__)
+from src.organizations.controller import controller
+
+#from src.database import db_session, read_json
+#from .model import Organization #, OrganizationSchema
+#from datetime import datetime
+#import json
+#from src import clogger
+#import flask
+
+
+@organization_pages.route("/")
+@organization_pages.route("")
+@organization_pages.route(".json")
+def index():
+    organizations=controller.get_all()
+    return jsonify(organizations=organizations)
+
+@organization_pages.route("/<int:id>",methods=['GET'])
+@organization_pages.route("/<int:id>.json",methods=['GET'])
+def get(id):
+    organization=controller.get(id)
+    return jsonify(organization=organization)
+
+@organization_pages.route("/<string:key>",methods=['GET'])
+@organization_pages.route("/<string:key>.json",methods=['GET'])
+def get_by_key(key):
+    organization=controller.get_by_key(key)
+    return jsonify(organization=organization)
+
+@organization_pages.route("/<string:key>/articles",methods=['GET'])
+@organization_pages.route("/<string:key>/articles.json",methods=['GET'])
+def get_articles_by_key(key):
+    organization=controller.get_by_key(key)
+    articles=controller.get_by_key_articles(key)
+    return jsonify(organization=organization, articles=articles)
+
+@organization_pages.route("/<int:id>/sections",methods=['GET'])
+@organization_pages.route("/<int:id>/sections.json",methods=['GET'])
+def get_sections(id):
+    organization=Organization.query.get(id)
+    sections=organization.sections
+    return jsonify(organization=organization, sections=sections)
+
+@organization_pages.route("/<int:id>/articles",methods=['GET'])
+@organization_pages.route("/<int:id>/articles.json",methods=['GET'])
+def get_articles(id):
+    articles=controller.get_articles(id)
+    organization=controller.get(id)
+    return jsonify(organization=organization, articles=articles)
+
+
+@organization_pages.route("/<int:id>",methods=['PUT'])
+@organization_pages.route("/<int:id>.json",methods=['PUT'])
+def update(id):
+    o,errors=controller.update(id,request)
+    return jsonify(organization=o, errors=errors)
+
+@organization_pages.route("/<int:id>",methods=['DELETE'])
+@organization_pages.route("/<int:id>.json",methods=['DELETE'])
+def delete(id):
+    controller.delete(id)
+    return jsonify()
+
+
+@organization_pages.route("/",methods=['POST'])
+@organization_pages.route("",methods=['POST'])
+@organization_pages.route(".json",methods=['POST'])
+def create():
+    organization,errors=controller.create(request)
+    return jsonify(organization=organization,errors=errors)

organizationtypes/controller.py

@@ -0,0 +1,13 @@
+from src.controller import BaseController
+
+from .model import Organizationtype
+class OrgController(BaseController):
+   __myclass__=Organizationtype
+   __jsonid__='organizationtype'
+ 
+#def get_all():
+#    Organizationtype.query.all()
+
+#def get(id):
+#    Organizationtype.query.get(id)
+controller=OrgController()

organizationtypes/model.py

@@ -0,0 +1,34 @@
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Text, ForeignKey, Index, TIMESTAMP
+from sqlalchemy.orm import relationship, validates
+
+from datetime import datetime
+from src.database import Base,db_session
+from marshmallow import Schema, fields, post_load, ValidationError
+from src import clogger
+import json
+import flask 
+from src.models import Section
+import re
+
+def validate_image(k, img):
+    return True
+class FullOrganizationtypeSchema(Schema):
+    id = fields.Integer()
+    name = fields.String()
+    image = fields.String()
+    updated_at = fields.DateTime()
+    created_at = fields.DateTime()
+
+class Organizationtype(Base):
+    __tablename__ = 'organizationtypes'
+    name=Column(String(250), unique=True, nullable=False)
+    organizations=relationship("Organization", back_populates="organizationtype")
+    image=Column(String(250))
+
+    __schema__= FullOrganizationtypeSchema
+    __whiteattrs__= ["url", "name", "image"]
+    __jsonattrs__=None # None means all
+
+    def __init__(self, data={}):
+        self.update(data,False)
+

organizationtypes/views.py

@@ -0,0 +1,50 @@
+from flask import Blueprint, jsonify, render_template, abort, redirect, url_for, request
+organizationtype_pages = Blueprint('organizationtypes', __name__)
+
+#from .model import Organizationtype #, Organizationtypeschema
+#from .model import SectionSchema
+#from datetime import datetime
+#import json
+#from src import clogger
+from src.organizationtypes.controller import controller
+#from src.database import db_session, read_json
+#import flask
+
+@organizationtype_pages.route("/")
+@organizationtype_pages.route("")
+@organizationtype_pages.route(".json")
+def index():
+    organizationtypes=controller.get_all()
+    return jsonify(organizationtypes=organizationtypes)
+
+@organizationtype_pages.route("/<int:id>",methods=['GET'])
+@organizationtype_pages.route("/<int:id>.json",methods=['GET'])
+def get(id):
+    organizationtype=controller.get(id)
+    return jsonify(organizationtype=organizationtype)
+
+@organizationtype_pages.route("/<int:id>",methods=['GET'])
+@organizationtype_pages.route("/<int:id>.json",methods=['GET'])
+def get_articles(id):
+    articles=controller.get_articles(id)
+    return jsonify(articles=articles)
+
+
+@organizationtype_pages.route("/<int:id>",methods=['PUT'])
+@organizationtype_pages.route("/<int:id>.json",methods=['PUT'])
+def update(id):
+    o,errors=controller.update(id,request)
+    return jsonify(organizationtype=o,errors=errors)
+
+
+@organizationtype_pages.route("/",methods=['POST'])
+@organizationtype_pages.route("",methods=['POST'])
+@organizationtype_pages.route(".json",methods=['POST'])
+def create():
+    organizationtype=controller.create(request)
+#    a=read_json(request)
+#    organizationtype=Organizationtype(a["organizationtype"])
+#    controller.
+#    db_session.add(organizationtype)
+#    db_session.commit()
+    return jsonify(organizationtype=organizationtype)

sections/model.py

@@ -10,34 +10,70 @@ import flask
 
 #from src.articles import Article
 
+class OrganizationtypeCompSchema(Schema):
+    id =fields.Integer()
+    name=fields.String()
+
+class OrganizationCompSchema(Schema):
+    id =fields.Integer()
+    name=fields.String()
+    organizationtype=fields.Nested(OrganizationtypeCompSchema)
+    
+class FullSectionSchema(Schema):
+    id=fields.Integer()
+    url =fields.String(allow_none=True )
+    crawlurl =fields.Integer(required=False,allow_none=True )
+    #published_date=fields.DateTime()
+    #date=fields.DateTime(allow_none=True)
+    name=fields.String(required=False,allow_none=True )
+    title=fields.String(dump_only=True)
+    foreign_name=fields.String()
+    group=fields.String(required=False,allow_none=True )
+    organization=fields.Nested(OrganizationCompSchema)
+    updated_at = fields.DateTime()
+    created_at = fields.DateTime()
+    
+    #    @post_load
+#    def make_section(self, data):
+#        return Section.deserialize(data)
+
+
+
 class Section(Base):
     __tablename__ = 'sections'
-    id = Column(Integer, primary_key=True)
+#    id = Column(Integer, primary_key=True)
     url = Column(String(250))
     crawlurl = Column(Integer)
     foreign_name = Column(String(250),unique=True)
     name=Column(String(250))
     group = Column(String(250))
+    organization_id=Column(Integer, ForeignKey('organizations.id'))
+    organization=relationship("Organization")
     articles=relationship("Article", back_populates="section")
-
+    __schema__=FullSectionSchema
-    def __json__(self):
+    __whiteattrs__=["name", "organization_id", "url"]
-        return SectionSchema().dump(self)[0]
+    __jsonattrs__ = ["name", "organization_id", "foreign_name", "title", "url", "name", "id", "organization.name", "organization.organizationtype.name", "updated_at"]
+    #    def __json__(self):
+#        return SectionSchema().dump(self)[0]
     def __init__(self, url=None,fname=None):
         self.url=url
         self.foreign_name=fname
 
+#    def dict(self):
+#        SectionSchema.dump(self)[0]
+        
     def title(self):
         t=self.name
         if t == None or t.strip()=="":
             t=self.foreign_name
         return t
-    @classmethod
+#    @classmethod
-    def deserialize(cls,data):
+#    def deserialize(cls,data):
-        a=Section()
+#        a=Section()
-        for c in Section.__table__.columns:
+#        for c in Section.__table__.columns:
-            if data.has_key(c.key):
+#            if data.has_key(c.key):
-                setattr(a, c.key,data[c.key])
+#                setattr(a, c.key,data[c.key])
-        return a
+#        return a
     
     @classmethod
     def find_or_create(cls, fname):
@@ -51,19 +87,6 @@ class Section(Base):
         db_session.commit()
         return s
 
-class FullSectionSchema(Schema):
-    id=fields.Integer()
-    url =fields.String()
-    crawlurl =fields.Integer()
-    #published_date=fields.DateTime()
-    #date=fields.DateTime(allow_none=True)
-    name=fields.String()
-    foreign_name=fields.String()
-    group=fields.String()
-    @post_load
-    def make_section(self, data):
-        return Article.deserialize(data)
-
     
 
 class ArticleCompSchema(Schema):
@@ -80,8 +103,8 @@ class ArticleCompSchema(Schema):
     section_id=fields.Integer()
 
 #from src.articles.model import ArticleCompSchema
-class SectionSchema(Schema):
+#class SectionSchema(Schema):
-    id=fields.Integer()
+#    id=fields.Integer()
-    foreign_name=fields.String()
+#    foreign_name=fields.String()
-    name=fields.String()
+#    name=fields.String()
 #    articles=fields.Nested(ArticleCompSchema,many=True)

sections/views.py

@@ -1,12 +1,12 @@
 from flask import Blueprint, jsonify, render_template, abort, redirect, url_for, request
 section_pages = Blueprint('sections', __name__)
 from .model import Section
-from .model import SectionSchema
+#from .model import SectionSchema
 #import flask
 from datetime import datetime
 import json
 from src import clogger
-import src.articles.controller as article_controller
+from src.articles import controller as article_controller
 from src.database import db_session, read_json
 import flask
 
@@ -23,8 +23,8 @@ def index():
 def update(id):
     section=Section.query.get(id)
     clogger.info(request.data)
-    a=request.get_json()
+    a=read_json(request)
-    section.name=a["name"]
+    section.update(a["section"])
     db_session.commit()
     return jsonify(section=section)