init commit

database.py

@@ -0,0 +1,55 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+from src import package_directory,clogger, cfg
+from os import path
+import json
+#engine = create_engine('sqlite:////home/andreas/www/crawler/test.db', convert_unicode=True)
+
+if cfg.get("db_path")==None or cfg.get("db_path").strip()=="":
+    db_path=package_directory
+else:
+    db_path=cfg.get("db_path")
+
+db_mainfile=cfg.get("db_mainfile")
+if db_mainfile == None or db_mainfile.strip()=="":
+    db_mainfile="../srctest.db"
+
+db_urlfile=cfg.get("db_mainfile")
+if db_urlfile == None or db_urlfile.strip()=="":
+    db_urlfile="../srctest_cu.db"
+
+    
+engine = create_engine('sqlite:///'+ path.join(db_path,db_mainfile), convert_unicode=True)
+
+db_session = scoped_session(sessionmaker(autocommit=False,
+                                         autoflush=False,
+                                         bind=engine))
+
+engine2 = create_engine('sqlite:///'+  path.join(db_path,db_urlfile), convert_unicode=True)
+
+db_session2 = scoped_session(sessionmaker(autocommit=False,
+                                         autoflush=False,
+                                          bind=engine2))
+
+Base = declarative_base()
+Base.query = db_session.query_property()
+Base2 = declarative_base()
+Base2.query = db_session2.query_property()
+
+def read_json(rq):
+    js=rq.get_json()
+    clogger.info(rq.data)
+    if js is None:
+        js=rq.form.to_dict()
+    if js=={} and rq.data != "":
+        js=json.loads(rq.data)    
+    return js
+
+def init_db():
+    import src.models    
+    Base.metadata.create_all(bind=engine)
+
+def init_db2():
+    from .compiler.models import CrawlUrl, CrawlCache
+    Base2.metadata.create_all(bind=engine2)