23 lines
615 B
Python
23 lines
615 B
Python
from .models import CrawlUrl
|
|
from .models import CrawlCache, CrawlCacheSchema
|
|
from src.database import db_session2,init_db,read_json,init_db2
|
|
from compiler import do_compile
|
|
from fetching import fetch_page
|
|
import mworker
|
|
|
|
def urls_test(id):
|
|
cu=CrawlUrl.query.get(id)
|
|
rw=fetch_page(cu.url)
|
|
h= {"url": cu.url, "sourcetype": cu.tpe, "raw": rw}
|
|
h2=do_compile(cu.tpe, h)
|
|
return {"rw": rw, "url": h, "comp": h2}
|
|
|
|
def urls_que(id):
|
|
cu=CrawlUrl.query.get(id)
|
|
mworker.queue_url(cu.tpe, cu.url)
|
|
return cu
|
|
|
|
def start_workers():
|
|
mworker.start_workers(1,1,1)
|
|
return "started workers"
|