bofh/fachschaften
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fetch all fb

Browse Source
This commit is contained in:
Andreas Stephanides
2017-02-17 09:58:35 +01:00
parent eb071d9f95
commit 80c42c04cd
3 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
compiler/fetching.py
View File

@@ -36,7 +36,7 @@ from datetime import datetime, timedelta
def fetch_page(furl):
def fetch_page(furl,p={}):
current_time = datetime.utcnow()
ten_weeks_ago = current_time - timedelta(days=cfg.cache_days)
u=urlparse.urlparse(furl)
@@ -49,7 +49,10 @@ def fetch_page(furl):
clogger.debug("fetching url: "+ str(furl))
if u[0]=='fb':
fb_time_since = str(int((current_time - timedelta(days=10)-datetime(1970,1,1)).total_seconds()))
furl=u[1]+u[2]+"?since="+fb_time_since+"&fields=story,created_time,id,message,attachments"
if p.has_key("nofollow") and p["nofollow"]==False:
furl=u[1]+u[2]+"?fields=story,created_time,id,message,attachments"
else:
furl=u[1]+u[2]+"?since="+fb_time_since+"&fields=story,created_time,id,message,attachments"
cc=CrawlCache.query.filter(CrawlCache.url==furl).filter(CrawlCache.fetched>ten_weeks_ago).first()
if cc is None:
tx = json.dumps(graph.get_object(id=furl))