first_commit

html_scrapy/elastic_publish.py

@@ -0,0 +1,25 @@
+import os
+from elasticsearch import Elasticsearch
+from searching import es_client, ELASTIC_INDEX
+
+def push_to_index(id, element):
+    element = check_elastic_document(element)
+    try:
+     with es_client() as client:
+      client.index(index=ELASTIC_INDEX, id=id,  document=element)
+    except Exception as e:
+        print(e)
+
+def check_elastic_document(element):
+    
+    for e in ["url", "title", "text", "published", "updated_at"]:
+        if not e in element:
+            raise AttributeError(f"A %s is needed for the Elastic Element" % e)
+    
+    return {    "published": str(element["published"]),
+                "text": element["text"],
+                "title": element["title"],
+                #"source": get_source(post),
+                "url": element["url"],
+                "updated_at": str(element["updated_at"])
+                }