bofh/fachschaften
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fbfeed + sectionreset

Browse Source
This commit is contained in:
Andreas Stephanides
2017-01-15 14:43:02 +01:00
parent 449a278d58
commit f475364213
7 changed files with 95 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
compiler/fetching.py
View File

@@ -40,10 +40,10 @@ def fetch_page(furl):
if u[0] == '':
furl=urlparse.urlunsplit(("http",u[1],u[2],u[3],u[4]))
cc=CrawlCache.query.filter(CrawlCache.url==furl).filter(CrawlCache.fetched>ten_weeks_ago).first()
if cc is None:
if cc is None or u[0]=='fb': # no caching for Facebook
clogger.debug("fetching url: "+ str(furl))
if u[0]=='fb':
tx = json.dumps(graph.get_object(id=u[1]+u[2]))
tx = json.dumps(graph.get_object(id=u[1]+u[2]+"?fields=story,created_time,id,message,attachments"))
else:
tx=s.get(furl).text
CrawlCache.store(furl,tx)