Merge pull request #175 from fedevergara/updates

Extraction of the mongodb connection for some plugins

Kahi_ranking_udea_works/kahi_ranking_udea_works/Kahi_ranking_udea_works.py

@@ -191,10 +191,10 @@ def parse_ranking_udea(reg, affiliation, empty_work):
     return entry
 
 
-def process_one(ranking_udea_reg, url, db_name, affiliation, empty_work, verbose=0):
-    client = MongoClient(url)
-    db = client[db_name]
-    collection = db["works"]
+def process_one(ranking_udea_reg, db, collection, affiliation, empty_work, verbose=0):
+    # client = MongoClient(url)
+    # db = client[db_name]
+    # collection = db["works"]
     doi = None
     # register has doi
     if ranking_udea_reg["DOI"]:
@@ -207,7 +207,7 @@ def process_one(ranking_udea_reg, url, db_name, affiliation, empty_work, verbose
             # updated
             for upd in colav_reg["updated"]:
                 if upd["source"] == "ranking_udea":
-                    client.close()
+                    # client.close()
                     return None  # Register already on db
                     # Could be updated with new information when ranking file updates
             entry = parse_ranking_udea(
@@ -371,7 +371,7 @@ def process_one(ranking_udea_reg, url, db_name, affiliation, empty_work, verbose
     else:  # does not have a doi identifier
         # elasticsearch section
         pass
-    client.close()
+    # client.close()
 
 
 class Kahi_ranking_udea_works(KahiBase):
@@ -444,19 +444,23 @@ def __init__(self, config):
                 {"names.name": "Universidad de Antioquia"})
 
     def process_ranking_udea(self):
-        Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            backend="threading")(
-            delayed(process_one)(
-                paper,
-                self.mongodb_url,
-                self.config["database_name"],
-                self.udea_reg,
-                self.empty_work(),
-                verbose=self.verbose
-            ) for paper in self.ranking
-        )
+        with MongoClient(self.mongodb_url) as client:
+            db = client[self.config["database_name"]]
+            collection = db["works"]
+
+            Parallel(
+                n_jobs=self.n_jobs,
+                verbose=self.verbose,
+                backend="threading")(
+                delayed(process_one)(
+                    paper,
+                    db,
+                    collection,
+                    self.udea_reg,
+                    self.empty_work(),
+                    verbose=self.verbose
+                ) for paper in self.ranking
+            )
 
     def run(self):
         self.process_ranking_udea()

Kahi_ranking_udea_works/kahi_ranking_udea_works/_version.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__version__ = '0.1.0-beta'
+__version__ = '0.1.1-beta'
 
 
 def get_version():

Kahi_scholar_works/kahi_scholar_works/Kahi_scholar_works.py

@@ -260,10 +260,10 @@ def parse_scholar(reg, empty_work, verbose=0):
     return entry
 
 
-def process_one(scholar_reg, url, db_name, empty_work, verbose=0):
-    client = MongoClient(url)
-    db = client[db_name]
-    collection = db["works"]
+def process_one(scholar_reg, db, collection, empty_work, verbose=0):
+    # client = MongoClient(url)
+    # db = client[db_name]
+    # collection = db["works"]
     doi = None
     # register has doi
     if scholar_reg["doi"]:
@@ -276,7 +276,7 @@ def process_one(scholar_reg, url, db_name, empty_work, verbose=0):
             # updated
             for upd in colav_reg["updated"]:
                 if upd["source"] == "scholar":
-                    client.close()
+                    # client.close()
                     return None  # Register already on db
                     # Could be updated with new information when scholar database changes
             entry = parse_scholar(
@@ -471,7 +471,7 @@ def process_one(scholar_reg, url, db_name, empty_work, verbose=0):
     else:  # does not have a doi identifier
         # elasticsearch section
         pass
-    client.close()
+    # client.close()
 
 
 class Kahi_scholar_works(KahiBase):
@@ -508,18 +508,23 @@ def __init__(self, config):
 
     def process_scholar(self):
         paper_list = list(self.scholar_collection.find())
-        Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            backend="threading")(
-            delayed(process_one)(
-                paper,
-                self.mongodb_url,
-                self.config["database_name"],
-                self.empty_work(),
-                verbose=self.verbose
-            ) for paper in paper_list
-        )
+
+        with MongoClient(self.mongodb_url) as client:
+            db = client[self.config["database_name"]]
+            collection = db["works"]
+
+            Parallel(
+                n_jobs=self.n_jobs,
+                verbose=self.verbose,
+                backend="threading")(
+                delayed(process_one)(
+                    paper,
+                    db,
+                    collection,
+                    self.empty_work(),
+                    verbose=self.verbose
+                ) for paper in paper_list
+            )
 
     def run(self):
         self.process_scholar()

Kahi_scholar_works/kahi_scholar_works/_version.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__version__ = '0.1.0-beta'
+__version__ = '0.1.1-beta'
 
 
 def get_version():

Kahi_scopus_works/kahi_scopus_works/Kahi_scopus_works.py

@@ -269,10 +269,10 @@ def parse_scopus(reg, empty_work, verbose=0):
     return entry
 
 
-def process_one(scopus_reg, url, db_name, empty_work, verbose=0):
-    client = MongoClient(url)
-    db = client[db_name]
-    collection = db["works"]
+def process_one(scopus_reg, db, collection, empty_work, verbose=0):
+    # client = MongoClient(url)
+    # db = client[db_name]
+    # collection = db["works"]
     doi = None
     # register has doi
     if scopus_reg["DOI"]:
@@ -285,7 +285,7 @@ def process_one(scopus_reg, url, db_name, empty_work, verbose=0):
             # updated
             for upd in colav_reg["updated"]:
                 if upd["source"] == "scopus":
-                    client.close()
+                    # client.close()
                     return None  # Register already on db
                     # Could be updated with new information when scopus database changes
             entry = parse_scopus(
@@ -475,7 +475,7 @@ def process_one(scopus_reg, url, db_name, empty_work, verbose=0):
     else:  # does not have a doi identifier
         # elasticsearch section
         pass
-    client.close()
+    # client.close()
 
 
 class Kahi_scopus_works(KahiBase):
@@ -513,18 +513,22 @@ def __init__(self, config):
 
     def process_scopus(self):
         paper_list = list(self.scopus_collection.find())
-        Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            backend="threading")(
-            delayed(process_one)(
-                paper,
-                self.mongodb_url,
-                self.config["database_name"],
-                self.empty_work(),
-                verbose=self.verbose
-            ) for paper in paper_list
-        )
+        with MongoClient(self.mongodb_url) as client:
+            db = client[self.config["database_name"]]
+            collection = db["works"]
+
+            Parallel(
+                n_jobs=self.n_jobs,
+                verbose=self.verbose,
+                backend="threading")(
+                delayed(process_one)(
+                    paper,
+                    db,
+                    collection,
+                    self.empty_work(),
+                    verbose=self.verbose
+                ) for paper in paper_list
+            )
 
     def run(self):
         self.process_scopus()

Kahi_scopus_works/kahi_scopus_works/_version.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__version__ = '0.1.0-beta'
+__version__ = '0.1.1-beta'
 
 
 def get_version():

Kahi_wos_works/kahi_wos_works/Kahi_wos_works.py

@@ -320,10 +320,10 @@ def parse_wos(reg, empty_work, verbose=0):
     return entry
 
 
-def process_one(wos_reg, url, db_name, empty_work, verbose=0):
-    client = MongoClient(url)
-    db = client[db_name]
-    collection = db["works"]
+def process_one(wos_reg, db, collection, empty_work, verbose=0):
+    # client = MongoClient(url)
+    # db = client[db_name]
+    # collection = db["works"]
     doi = None
     # register has doi
     if wos_reg["DI"]:
@@ -336,7 +336,7 @@ def process_one(wos_reg, url, db_name, empty_work, verbose=0):
             # updated
             for upd in colav_reg["updated"]:
                 if upd["source"] == "wos":
-                    client.close()
+                    # client.close()
                     return None  # Register already on db
                     # Could be updated with new information when wos database changes
             entry = parse_wos(
@@ -538,7 +538,7 @@ def process_one(wos_reg, url, db_name, empty_work, verbose=0):
     else:  # does not have a doi identifier
         # elasticsearch section
         pass
-    client.close()
+    # client.close()
 
 
 class Kahi_wos_works(KahiBase):
@@ -574,18 +574,23 @@ def __init__(self, config):
 
     def process_wos(self):
         paper_list = list(self.wos_collection.find())
-        Parallel(
-            n_jobs=self.n_jobs,
-            verbose=self.verbose,
-            backend="threading")(
-            delayed(process_one)(
-                paper,
-                self.mongodb_url,
-                self.config["database_name"],
-                self.empty_work(),
-                verbose=self.verbose
-            ) for paper in paper_list
-        )
+
+        with MongoClient(self.mongodb_url) as client:
+            db = client[self.config["database_name"]]
+            collection = db["works"]
+
+            Parallel(
+                n_jobs=self.n_jobs,
+                verbose=self.verbose,
+                backend="threading")(
+                delayed(process_one)(
+                    paper,
+                    db,
+                    collection,
+                    self.empty_work(),
+                    verbose=self.verbose
+                ) for paper in paper_list
+            )
 
     def run(self):
         self.process_wos()

Kahi_wos_works/kahi_wos_works/_version.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__version__ = '0.1.0-beta'
+__version__ = '0.1.1-beta'
 
 
 def get_version():