bot upgrade 1

compiler/controller.py

@@ -0,0 +1,22 @@
+from .models import CrawlUrl
+from .models import CrawlCache, CrawlCacheSchema
+from src.database import db_session2,init_db,read_json,init_db2
+from compiler import do_compile
+from fetching import fetch_page
+import mworker
+
+def urls_test(id):
+    cu=CrawlUrl.query.get(id)
+    rw=fetch_page(cu.url)
+    h= {"url": cu.url, "sourcetype": cu.tpe, "raw": rw}
+    h2=do_compile(cu.tpe, h)
+    return {"rw": rw, "url": h, "comp": h2}
+
+def urls_que(id):
+    cu=CrawlUrl.query.get(id)
+    mworker.queue_url(cu.tpe, cu.url)
+    return cu
+    
+def start_workers():
+    mworker.start_workers(1,1,1)
+    return "started workers"