Misc updates

doc/deployer/Migrating_SQLite_DB_to_PostgreSQL.txt

@@ -112,6 +112,12 @@ alter sequence auth_user_id_seq restart
 # Tabing BACKUP/DUMP of all DBs and table
 pg_dumpall > file-name.sql
 
+-- OR --
+echo "pg_dumpall > /tmp/file-name.sql" | sudo su - postgres
+# In above case, permission denied exception might gets raised to create file under `/tmp/`
+-- OR --
+echo "pg_dumpall > file-name.sql" | sudo su - postgres
+# In above case file-name.sql will be created under `/var/lib/postgresql/` folder
 
 # DROP database
 $ sudo su - postgres

doc/deployer/es_initialize.py

@@ -0,0 +1,133 @@
+# from bson import json_util
+# import os
+# import sys
+# import json
+# from elasticsearch import Elasticsearch
+# from gnowsys_ndf.ndf.models import *
+
+# from gnowsys_ndf.settings import GSTUDIO_SITE_NAME
+# from gnowsys_ndf.local_settings import *
+
+
+# ##### use the below commented lines if you are working with Python 2.x   #####
+# # reload(sys)  
+# # sys.setdefaultencoding('UTF8')
+
+# es = Elasticsearch(GSTUDIO_ELASTIC_SEARCH_PROTOCOL+"://"+GSTUDIO_ELASTIC_SEARCH_SUPERUSER+":"+GSTUDIO_ELASTIC_SEARCH_SUPERUSER_PASSWORD+"@"+GSTUDIO_ELASTIC_SEARCH_ALIAS+":"+GSTUDIO_ELASTIC_SEARCH_PORT,timeout=100, retry_on_timeout=True)
+
+
+# author_index = "author_" + GSTUDIO_SITE_NAME.lower()
+# index = GSTUDIO_SITE_NAME.lower()
+# gsystemtype_index = "node_type_" + GSTUDIO_SITE_NAME.lower()
+
+# page_id = 0 
+
+# def index_docs(all_docs):
+# 	k = 0
+# 	all_docs_count = all_docs.count()
+# 	for node in all_docs:
+# 		print "[ %s/%s ] : %s " % (k, all_docs_count, node._id)
+# 		if(node._type == "GSystem"):
+# 			try:
+# 				node.get_neighbourhood(node.member_of)
+# 			except Exception as e:
+# 				print(e)
+# 			else:
+# 				pass
+
+# 		doc = json.dumps(node, cls=NodeJSONEncoder) #convert mongoDB object to a JSON string
+# 		#doc = json.loads(doc,object_hook=json_util.object_hook) ->get back mongoDB object from JSON string
+
+# 		document = json.loads(doc) #convert JSON string to a python dictionary
+# 		#doc = json.dumps(document) #convert python dictionary to JSON string
+
+# 		if("name" in document.keys()):
+
+# 			# replacing "_" starting keys:
+# 			document["id"] = document.pop("_id")
+# 			document["type"] = document.pop("_type")
+
+# 			if("object_type" in document.keys()):
+# 				document["object_type"] = str(document["object_type"])
+# 			if("property_order" in document.keys()):
+# 				document["property_order"] = str(document["property_order"])
+# 			if("object_value" in document.keys()):
+# 				document["object_value"] = str(document["object_value"])	#for consistent mapping 
+
+# 			doc_type = get_document_type(document)
+# 			es.index(index=index, doc_type=doc_type, id=document["id"], body=document)
+# 			if "contributors" in document.keys():
+# 				contributors = document["contributors"]
+# 				for contributor_id in contributors:
+# 					es.index(index = author_index, doc_type = contributor_id, id=document["id"], body = document)
+
+# 			if(document["type"] == "GSystem"):
+# 				for type_ids in document['member_of']:
+# 					es.index(index = gsystemtype_index, doc_type = type_ids, id = document["id"], body = document)
+
+# 			k+=1
+
+# def get_document_type(document):
+# 	types_arr = ["Author", "GAttribute", "GRelation", "AttributeType", "Filehive", "RelationType", "Group", "GSystemType"]	
+# 	if document["type"]=="GSystem":
+# 		for ids in document['member_of']:  #document is a member of the Page GSystemType
+# 			if(ids == page_id):
+# 				return "Page"
+# 		if('if_file' in document.keys()):
+# 			if(document["if_file"]["mime_type"] is not None):
+# 				data = document["if_file"]["mime_type"].split("/")
+# 				doc_type = data[0]
+# 			else:
+# 				doc_type = "NotMedia"
+# 		else:
+# 			doc_type = "NotMedia"
+# 	elif (document["type"] in types_arr):
+# 		doc_type = document["type"]
+# 	else:
+# 		doc_type = "DontCare"
+# 	return doc_type
+
+
+# def main():
+
+# 	print("Starting the indexing process...")
+# 	# DELETING existing/old indexes
+# 	if(es.indices.exists(index)):
+# 		print("Deleting the existing index: "+index+" for reindexing")
+# 		res = es.indices.delete(index=index)
+# 		print("The delete response is %s " % res)
+
+# 	if(es.indices.exists(author_index)):
+# 		print("Deleting the existing author_index: "+author_index +" for reindexing")
+# 		res = es.indices.delete(index=author_index)
+# 		print("The delete response is %s " % res)
+
+# 	if(es.indices.exists(gsystemtype_index)):
+# 		print("Deleting the existing gtype index: "+ index+ "for reindexing")
+# 		res = es.indices.delete(index=gsystemtype_index)
+# 	# --- END of DELETING existing/old indexes
+
+# 	with open("/home/docker/code/gstudio/gnowsys-ndf/gnowsys_ndf/req_body.json") as req_body:
+# 		request_body = json.load(req_body)
+# 	with open("/home/docker/code/gstudio/gnowsys-ndf/gnowsys_ndf/req_body_gtype.json") as req_body_type:
+# 		request_body_gtype = json.load(req_body_type)
+# 	with open(GSTUDIO_DOCUMENT_MAPPING+"/gsystemtype_map.json") as gtypemap:
+# 		gtype_map = json.load(gtypemap)
+# 	page_id = gtype_map["Page"]
+
+# 	res = es.indices.create(index=index, body=request_body)
+# 	print("Response for index creation")
+# 	print(res)
+
+# 	res = es.indices.create(index=gsystemtype_index, body=request_body_gtype)
+# 	print("Response for index creation")
+# 	print(res)
+
+# 	res = es.indices.create(index=author_index, body=request_body_gtype)
+# 	print("Response for index creation")
+# 	print(res)
+
+# 	all_docs = node_collection.find(no_cursor_timeout=True).batch_size(5)
+# 	index_docs(all_docs)
+
+# main()

doc/deployer/es_injection.py

@@ -0,0 +1,204 @@
+from bson import json_util
+import os
+import sys
+import json
+from elasticsearch import Elasticsearch
+from gnowsys_ndf.ndf.models import *
+
+#from gnowsys_ndf.settings import GSTUDIO_SITE_NAME
+#from gnowsys_ndf.local_settings import GSTUDIO_DOCUMENT_MAPPING
+from gnowsys_ndf.settings import *
+
+try:
+    from bson import ObjectId
+except ImportError:  # old pymongo
+    from pymongo.objectid import ObjectId
+##### use the below commented lines if you are working with Python 2.x   #####
+# reload(sys)
+# sys.setdefaultencoding('UTF8')
+
+es = Elasticsearch(GSTUDIO_ELASTIC_SEARCH_PROTOCOL+"://"+GSTUDIO_ELASTIC_SEARCH_SUPERUSER+":"+GSTUDIO_ELASTIC_SEARCH_SUPERUSER_PASSWORD+"@"+GSTUDIO_ELASTIC_SEARCH_ALIAS+":"+GSTUDIO_ELASTIC_SEARCH_PORT,timeout=100, retry_on_timeout=True)
+
+#author_index = "author_" + GSTUDIO_SITE_NAME.lower()
+#index = GSTUDIO_SITE_NAME.lower()
+#gsystemtype_index = "node_type_" + GSTUDIO_SITE_NAME.lower()
+with open("/home/docker/code/gstudio/gnowsys-ndf/gnowsys_ndf/gstudio_configs/req_body.json") as req_body:
+    request_body = json.load(req_body)
+with open("/home/docker/code/gstudio/gnowsys-ndf/gnowsys_ndf/gstudio_configs/triples.json") as triples:
+    triples_body = json.load(triples)
+
+
+page_id = 0
+
+
+def index_docs(all_docs,index,doc_type):
+    k = 0
+    all_docs_count = all_docs.count()
+    for docs in all_docs:
+        print "[ %s/%s ] : %s " % (k, all_docs_count, docs._id)
+
+
+        doc = json.dumps(docs, cls=NodeJSONEncoder)  # convert mongoDB object to a JSON string
+        # doc = json.loads(doc,object_hook=json_util.object_hook) ->get back mongoDB object from JSON string
+
+        document = json.loads(doc)  # convert JSON string to a python dictionary
+        # doc = json.dumps(document) #convert python dictionary to JSON string
+        document["id"] = document.pop("_id")
+        document["type"] = document.pop("_type")
+
+        #for docs in doc_type:
+        print document["type"]
+        #print document
+        doc_type = document["type"].lower()
+        es.index(index=index, doc_type=doc_type, id=document["id"], body=document)
+
+        k += 1
+
+    #file_name.close()
+
+
+
+def get_document_type(document):
+
+	if document["type"]=="GSystem":
+		#for ids in document['member_of']:  #document is a member of the Page GSystemType
+			#if(ids == page_id):
+				#return "Page"
+		if('if_file' in document.keys()):
+			if(document["if_file"]["mime_type"] is not None):
+				data = document["if_file"]["mime_type"].split("/")
+				doc_type = data[0]
+			else:
+				doc_type = "notmedia"
+		else:
+			doc_type = "notmedia"
+
+	else:
+		doc_type = "dontcare"
+	return doc_type
+
+
+
+def main():
+    #f = open("/data/nodes.txt", "w")
+    #os.chmod("/data/nodes.txt", 0o777)
+
+    nodes = {}
+    triples = {}
+    benchmarks = {}
+    filehives = {}
+    buddys = {}
+    counters = {}
+
+    #all_docs = [ triples, buddys, benchmarks, nodes, counters]
+
+    print("Starting the indexing process...")
+
+    for index, doc_type in GSTUDIO_ELASTIC_SEARCH_INDEX.items():
+        temp = []
+
+
+        index_lower = index.lower()
+        if (not es.indices.exists(index_lower)):
+            if (index_lower == "triples"):
+                res = es.indices.create(index=index_lower, body=triples_body)
+            else:
+                res = es.indices.create(index=index_lower, body=request_body)
+
+        if (es.indices.exists(index_lower)):
+
+            res = es.search(index=index_lower, body={"query": {"match_all": {}}, "_source": ["id"]}, scroll="1m", size="10")
+
+            scrollid = res['_scroll_id']
+
+            while len(res['hits']['hits']) > 0:
+
+                for hit in res['hits']['hits']:
+                    # print(hit["_source"]["id"])
+                    # f.write(hit["_source"]["id"] + '\n')
+                    # res = es.search(index="nodes", body={"scroll_id": '"' + scrollid + '"'}, search_type="scan",
+                    #           scroll="1m", size="10")
+                    temp.append(ObjectId(hit["_source"]["id"]))
+
+                res = es.scroll(scrollid, scroll="1m")
+
+
+            if(index_lower == "nodes"):
+                nodes = node_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+
+                if(nodes.count() == 0):
+                    print("All "+ index_lower +" documents has injected to elasticsearch")
+                    continue
+                else:
+                    index_docs(nodes, index_lower, doc_type)
+
+            elif (index_lower == "triples"):
+                triples = triple_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+                if (triples.count() == 0):
+                    print("All " + index_lower + " documents has injected to elasticsearch")
+                    continue
+                else:
+                    # f = open("/data/triples.txt", "w")
+                    # os.chmod("/data/triples.txt", 0o777)
+                    index_docs(triples, index_lower, doc_type)
+
+            elif (index_lower == "benchmarks"):
+                benchmarks = benchmark_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+                if (benchmarks.count() == 0):
+                    print("All " + index_lower + " documents has injected to elasticsearch")
+                    continue
+                else:
+                    index_docs(benchmarks, index_lower, doc_type)
+
+            elif (index_lower == "filehives"):
+                filehives = filehive_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+                if (filehives.count() == 0):
+                    print("All " + index_lower + " documents has injected to elasticsearch")
+                    continue
+                else:
+                    index_docs(filehives, index_lower, doc_type)
+
+            elif (index_lower == "buddies"):
+                buddys = buddy_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+                if (buddys.count() == 0):
+                    print("All " + index_lower + " documents has injected to elasticsearch")
+                    continue
+                else:
+                    index_docs(buddys, index_lower, doc_type)
+
+            elif (index_lower == "counters"):
+                counters = counter_collection.find({ '_id': {'$nin': temp} }).batch_size(5)
+                if (counters.count() == 0):
+                    print("All " + index_lower + " documents has injected to elasticsearch")
+                    continue
+                else:
+                    index_docs(counters, index_lower, doc_type)
+
+
+            #print(res['_scroll_id'])
+            #print(res['hits']['total'])
+
+                #f = open("/data/nodes.txt", "w+")
+                #os.chmod("/data/nodes.txt", 0o777)
+
+                   # f = open("/data/nodes.txt", "w+")
+                    # os.chmod("/data/nodes.txt", 0o777)
+
+                    # es.scroll(scrollid, scroll="1m")
+
+                    #print(temp)
+
+                    # DELETING existing/old indexes
+
+                    # for index in GSTUDIO_ELASTIC_SEARCH_INDEX.keys():
+                    # if (es.indices.exists(index_lower)):
+                    #   print("Deleting the existing index: " + index.lower() + " for reindexing")
+                    #   res = es.indices.delete(index=index.lower())
+                    #   print("The delete response is %s " % res)
+
+
+                    # --- END of DELETING existing/old indexes
+
+
+
+main()