merge

compiler/mprocess.py

@@ -5,7 +5,7 @@ from src.database import db_session
 from mqueues import fetch_queue, compile_queue, put_fetch_queue
 from fetching import fetch_page, downloadfile, announce_articleid
 from fixing import fix_html, fix_file
-#from src import app
+from sqlalchemy.exc import InvalidRequestError
 
 from compiler import article_types
 from fixing import fix_link
@@ -20,20 +20,20 @@ def process_article(art):
         clogger.error("Invalid article hash:" + str(art))
         aa=None
     else:
-        art["text"]=fix_html(art["text"],art["url"])
+        art["text"] = fix_html(art["text"], art["url"])
         if "image" in art:
             art["image"]=fix_file(art["url"], art["image"])
         clogger.info(art)
         aa = Article.from_hash(art)
         aa.process_hash(art)
-        aa.last_fetched=datetime.now()
-        aa.sourcetype=art["sourcetype"]
+        aa.last_fetched = datetime.now()
+        aa.sourcetype = art["sourcetype"]
         db_session.add(aa)
         try:
             db_session.commit()
         except InvalidRequestError,e:
             db_session.rollback()
-            clogger.error(e)    
+            clogger.error(e)
         clogger.info("Updated/Added Article "+ str(aa.id) + ": " + (aa.title.encode("utf-8")))
     return aa
 #        app.logger.info("Updated/Added Article "+ str(aa.id) + ": " + (aa.title.encode("utf-8")))
@@ -92,4 +92,5 @@ def do_process(tpe, cont,params={}):
                 if a.has_key("url")==False:
                     a["url"]=cont["url"]
                 process_article(a)
+    db_session.remove()
     return