integrating entities and several fixes

Kahi_minciencias_opendata_events/kahi_minciencias_opendata_events/process_one.py

@@ -6,6 +6,53 @@
 from re import search
 
 
+def get_units_affiations(db, author_db, affiliations):
+    """
+    Method to get the units of an author in a register. ex: faculty, department and group.
+
+    Parameters:
+    ----------
+    db : pymongo.database.Database
+        Database connection to colav database.
+    author_db : dict
+        record from person
+    affiliations : list
+        list of affiliations from the parse_minciencias_opendata method
+
+    Returns:
+    -------
+    list
+        list of units of an author (entries from using affiliations)
+    """
+    institution_id = None
+    # verifiying univeristy
+    for j, aff in enumerate(affiliations):
+        aff_db = db["affiliations"].find_one(
+            {"_id": aff["id"]}, {"_id": 1, "types": 1})
+        if aff_db:
+            types = [i["type"] for i in aff_db["types"]]
+            if "group" in types or "department" in types or "faculty" in types:
+                aff_db = None
+                continue
+        if aff_db:
+            count = db["person"].count_documents(
+                {"_id": author_db["_id"], "affiliations.id": aff_db["_id"]})
+            if count > 0:
+                institution_id = aff_db["_id"]
+                break
+    units = []
+    for aff in author_db["affiliations"]:
+        if aff["id"] == institution_id:
+            continue
+        count = db["affiliations"].count_documents(
+            {"_id": aff["id"], "relations.id": institution_id})
+        if count > 0:
+            types = [i["type"] for i in aff["types"]]
+            if "department" in types or "faculty" in types:
+                units.append(aff)
+    return units
+
+
 def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, verbose=0):
     """
     Method to update a register in the kahi database from minciencias opendata database if it is found.
@@ -71,10 +118,7 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
                     group_id = minciencias_author["affiliations"][0]['external_ids'][0]['id']
 
                     affiliations_db = db["affiliations"].find_one(
-                        {"external_ids.source": "scienti", "external_ids.id": group_id})
-                    if not affiliations_db:
-                        affiliations_db = db["affiliations"].find_one(
-                            {"external_ids.id": group_id})
+                        {"external_ids.id": group_id})
 
                     if affiliations_db:
                         for i, author in enumerate(colav_reg["authors"]):
@@ -178,6 +222,12 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
                             if "education" in types:
                                 if relation["id"] not in affs:
                                     author["affiliations"].append(relation)
+                    aff_units = get_units_affiations(
+                        db, author_db, author["affiliations"])
+                    for aff_unit in aff_units:
+                        if aff_unit not in author["affiliations"]:
+                            author["affiliations"].append(aff_unit)
+
                     break
 
     collection.update_one(
@@ -239,10 +289,8 @@ def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, ve
                     if minciencias_author["affiliations"]:
                         group_id = minciencias_author["affiliations"][0]['external_ids'][0]['id']
                         affiliations_db = db["affiliations"].find_one(
-                            {"external_ids.source": "scienti", "external_ids.id": group_id})
-                        if not affiliations_db:
-                            affiliations_db = db["affiliations"].find_one(
-                                {"external_ids.id": group_id})
+                            {"external_ids.id": group_id})
+                        if affiliations_db:
                             if entry['authors'][0]['external_ids'][0]['id'] == ext['id']:
                                 entry['authors'][0]["affiliations"].append(
                                     {
@@ -271,8 +319,35 @@ def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, ve
     group_id = openadata_reg["cod_grupo_gr"]
     rgroup = db["affiliations"].find_one({"external_ids.id": group_id})
     if rgroup:
-        entry["groups"].append(
-            {"id": rgroup["_id"], "name": rgroup["names"][0]["name"]})
+        found = False
+        for group in entry["groups"]:
+            if group["id"] == rgroup["_id"]:
+                found = True
+                break
+        if not found:
+            entry["groups"].append(
+                {"id": rgroup["_id"], "name": rgroup["names"][0]["name"]})
+
+        # Adding group relation affiliation to the author affiliations
+        if author_db and rgroup["relations"]:
+            for author in entry["authors"]:
+                if author["id"] == author_db["_id"]:
+                    affs = [aff["id"] for aff in author["affiliations"]]
+                    for relation in rgroup["relations"]:
+                        types = []
+                        if "types" in relation.keys() and relation["types"]:
+                            types = [rel["type"].lower()
+                                     for rel in relation["types"]]
+                            if "education" in types:
+                                if relation["id"] not in affs:
+                                    author["affiliations"].append(relation)
+                    aff_units = get_units_affiations(
+                        db, author_db, author["affiliations"])
+                    for aff_unit in aff_units:
+                        if aff_unit not in author["affiliations"]:
+                            author["affiliations"].append(aff_unit)
+
+                    break
 
     # insert in mongo
     collection.insert_one(entry)

Kahi_minciencias_opendata_patents/kahi_minciencias_opendata_patents/process_one.py

@@ -4,7 +4,54 @@
 from re import search
 
 
-def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, verbose=0):
+def get_units_affiations(db, author_db, affiliations):
+    """
+    Method to get the units of an author in a register. ex: faculty, department and group.
+
+    Parameters:
+    ----------
+    db : pymongo.database.Database
+        Database connection to colav database.
+    author_db : dict
+        record from person
+    affiliations : list
+        list of affiliations from the parse_minciencias_opendata method
+
+    Returns:
+    -------
+    list
+        list of units of an author (entries from using affiliations)
+    """
+    institution_id = None
+    # verifiying univeristy
+    for j, aff in enumerate(affiliations):
+        aff_db = db["affiliations"].find_one(
+            {"_id": aff["id"]}, {"_id": 1, "types": 1})
+        if aff_db:
+            types = [i["type"] for i in aff_db["types"]]
+            if "group" in types or "department" in types or "faculty" in types:
+                aff_db = None
+                continue
+        if aff_db:
+            count = db["person"].count_documents(
+                {"_id": author_db["_id"], "affiliations.id": aff_db["_id"]})
+            if count > 0:
+                institution_id = aff_db["_id"]
+                break
+    units = []
+    for aff in author_db["affiliations"]:
+        if aff["id"] == institution_id:
+            continue
+        count = db["affiliations"].count_documents(
+            {"_id": aff["id"], "relations.id": institution_id})
+        if count > 0:
+            types = [i["type"] for i in aff["types"]]
+            if "department" in types or "faculty" in types:
+                units.append(aff)
+    return units
+
+
+def process_one_update(openadata_reg, colav_reg, db, collection, empty_patent, verbose=0):
     """
     Method to update a register in the kahi database from minciencias opendata database if it is found.
     This means that the register is already on the kahi database and it is being updated with new information.
@@ -20,13 +67,13 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
         Database where the colav collections are stored, used to search for authors and affiliations.
     collection : pymongo.collection.Collection
         Collection in the database where the register is stored (Collection of patents)
-    empty_work : dict
+    empty_patent : dict
         Empty dictionary with the structure of a register in the database
     verbose : int, optional
         Verbosity level. The default is 0.
     """
     entry = parse_minciencias_opendata(
-        openadata_reg, empty_event.copy(), verbose=verbose)
+        openadata_reg, empty_patent.copy(), verbose=verbose)
     # updated
     for upd in colav_reg["updated"]:
         if upd["source"] == "minciencias":
@@ -69,10 +116,7 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
                     group_id = minciencias_author["affiliations"][0]['external_ids'][0]['id']
 
                     affiliations_db = db["affiliations"].find_one(
-                        {"external_ids.source": "scienti", "external_ids.id": group_id})
-                    if not affiliations_db:
-                        affiliations_db = db["affiliations"].find_one(
-                            {"external_ids.id": group_id})
+                        {"external_ids.id": group_id})
 
                     if affiliations_db:
                         for i, author in enumerate(colav_reg["authors"]):
@@ -176,6 +220,12 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
                             if "education" in types:
                                 if relation["id"] not in affs:
                                     author["affiliations"].append(relation)
+                    aff_units = get_units_affiations(
+                        db, author_db, author["affiliations"])
+                    for aff_unit in aff_units:
+                        if aff_unit not in author["affiliations"]:
+                            author["affiliations"].append(aff_unit)
+
                     break
 
     collection.update_one(
@@ -191,7 +241,7 @@ def process_one_update(openadata_reg, colav_reg, db, collection, empty_event, ve
     )
 
 
-def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, verbose=0):
+def process_one_insert(openadata_reg, db, collection, empty_patent, es_handler, verbose=0):
     """
     Function to insert a new register in the database if it is not found in the colav(kahi patents) database.
     This means that the register is not on the database and it is being inserted.
@@ -210,15 +260,15 @@ def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, ve
         Database where the colav collections are stored, used to search for authors and affiliations.
     collection : pymongo.collection.Collection
         Collection in the database where the register is stored (Collection of patents)
-    empty_work : dict
+    empty_patent : dict
         Empty dictionary with the structure of a register in the database
     es_handler : Similarity
         Elasticsearch handler to insert the register in the elasticsearch index, Mohan's Similarity class.
     verbose : int, optional
         Verbosity level. The default is 0.
     """
     # parse
-    entry = parse_minciencias_opendata(openadata_reg, empty_work.copy())
+    entry = parse_minciencias_opendata(openadata_reg, empty_patent.copy())
     # search authors and affiliations in db
     # authors
     minciencias_author = ""
@@ -237,10 +287,8 @@ def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, ve
                     if minciencias_author["affiliations"]:
                         group_id = minciencias_author["affiliations"][0]['external_ids'][0]['id']
                         affiliations_db = db["affiliations"].find_one(
-                            {"external_ids.source": "scienti", "external_ids.id": group_id})
-                        if not affiliations_db:
-                            affiliations_db = db["affiliations"].find_one(
-                                {"external_ids.id": group_id})
+                            {"external_ids.id": group_id})
+                        if affiliations_db:
                             if entry['authors'][0]['external_ids'][0]['id'] == ext['id']:
                                 entry['authors'][0]["affiliations"].append(
                                     {
@@ -269,8 +317,35 @@ def process_one_insert(openadata_reg, db, collection, empty_work, es_handler, ve
     group_id = openadata_reg["cod_grupo_gr"]
     rgroup = db["affiliations"].find_one({"external_ids.id": group_id})
     if rgroup:
-        entry["groups"].append(
-            {"id": rgroup["_id"], "name": rgroup["names"][0]["name"]})
+        found = False
+        for group in entry["groups"]:
+            if group["id"] == rgroup["_id"]:
+                found = True
+                break
+        if not found:
+            entry["groups"].append(
+                {"id": rgroup["_id"], "name": rgroup["names"][0]["name"]})
+
+        # Adding group relation affiliation to the author affiliations
+        if author_db and rgroup["relations"]:
+            for author in entry["authors"]:
+                if author["id"] == author_db["_id"]:
+                    affs = [aff["id"] for aff in author["affiliations"]]
+                    for relation in rgroup["relations"]:
+                        types = []
+                        if "types" in relation.keys() and relation["types"]:
+                            types = [rel["type"].lower()
+                                     for rel in relation["types"]]
+                            if "education" in types:
+                                if relation["id"] not in affs:
+                                    author["affiliations"].append(relation)
+                    aff_units = get_units_affiations(
+                        db, author_db, author["affiliations"])
+                    for aff_unit in aff_units:
+                        if aff_unit not in author["affiliations"]:
+                            author["affiliations"].append(aff_unit)
+
+                    break
 
     # insert in mongo
     collection.insert_one(entry)