copy develop to master working tree

Author: albsantosdel

LICENSE.rst

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Mann Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/getting_started/create-new-project.rst

@@ -47,15 +47,15 @@ This will print some warnings, which should be okay
 
 Project creation
 ----------------
+
 **Browser**
 
-Go to ``http://localhost:5000/apps/projectCreation``
+Navigate to project creation from the front page by pressing the button "PROJECT CREATION"
+or go to ``http://localhost:5000/apps/projectCreation``
 
-This will direct you to the project creation page.
 Fill in your project information.
-Remember to press "Add" after selecting items from the drop-down menu (the page will update every time).
 Press "Create Project" (you will now be able to find your project in the neo4j database)
-Download the "Clinical Data template" and proceed to the 'Upload project experimental data'_.
+Download the "Clinical Data template" and proceed to 'Upload project experimental data'_.
 
 
 Other

docs/getting_started/upload-data.rst

@@ -36,14 +36,15 @@ Follow this link to search for 'SNOWMED'_ terms:
 .. _SNOWMED: https://browser.ihtsdotools.org/?perspective=full&conceptId1=734000001&edition=MAIN/2019-07-31&release=&languages=en
 Example: To add a column with "Age" search for "age". This gives multiple matches, with the first one being: "Age (qualifier value), SCTID:397669002". Please enter this information as column header with the SCTID in brackets: Age (qualifier value) (397669002)
 
-If you can't find your header in SNOWMED????
+If you can't find your header in SNOWMED you should contact annelaura.bach@cpr.ku.dk, put "Header Creation, CKG" in the subject.
+In the email please provide your "missing" header and a description of what it is.
 
 **Proteomic data**
-
-Do not perform any imputations or similar methods on your data before uploading it. This will be carried out in the CKG.
+Do not perform any imputations or similar on your data before uploading it. This will be carried out by the CKG.
 
 Data from MaxQuant(DDA):
 Upload the proteinGroups.txt
+Upload
 
 Data from Spectronaut(DIA):
 ?

requirements.txt

@@ -27,16 +27,10 @@ cvxpy==1.0.24
 cycler==0.10.0
 cyjupyter==0.2.0
 dabest==0.2.4
+dash-bio==0.4.5
 dash==1.2.0
 dash-auth==1.3.2
-dash-bio==0.1.1
-dash-bootstrap-components==0.7.1
-dash-core-components==1.1.2
 dash-cytoscape==0.1.1
-dash-html-components==1.0.1
-dash-renderer==1.0.1
-dash-table==4.2.0
-dask==2.0.0
 decorator==4.4.0
 defusedxml==0.6.0
 dill==0.2.9
@@ -211,3 +205,6 @@ bcrypt==3.1.7
 et-xmlfile==1.0.1
 jdcal==1.4.1
 openpyxl==3.0.1
+watchdog>=0.8.3
+sphinx-rtd-theme>=0.4.3
+recommonmark>=0.6.0

src/analytics_core/utils.py

@@ -45,12 +45,13 @@ def generate_html(network):
                                     conf=network.conf,
                                     tooltip_link=use_link_template)
 
-def neoj_path_to_networkx(paths, key='path'):
+def neo4j_path_to_networkx(paths, key='path'):
     regex = r"\(?(.+)\)\<?\-\>?\[\:(.+)\s\{.*\}\]\<?\-\>?\((.+)\)?"
     nodes = set()
     rels = set()
     for r in paths:
-        path = str(r[key])
+        if key is not None:
+            path = str(r[key])
         matches = re.search(regex, path)
         if matches:
             source = matches.group(1)
@@ -73,6 +74,37 @@ def neoj_path_to_networkx(paths, key='path'):
 
     return G
 
+def neo4j_schema_to_networkx(schema):
+    regex = r"\(?(.+)\)\<?\-\>?\[\:(.+)\s\{.*\}\]\<?\-\>?\((.+)\)"
+    nodes = set()
+    rels = set()
+    if 'relationships' in schema[0]:
+        relationships = schema[0]['relationships']
+        for relationship in relationships:
+            matches = re.search(regex, str(relationship))
+            if matches:
+                source = matches.group(1)
+                source_match = re.search(regex, source)
+                if source_match:
+                    source = source_match.group(1)
+                    relationship = source_match.group(2)
+                    target = source_match.group(3)
+                    nodes.update([source, target])
+                    rels.add((source, target, relationship))
+                    source = target
+                relationship = matches.group(2)
+                target = matches.group(3)
+                nodes.update([source, target])
+                rels.add((source, target, relationship))
+    G = nx.Graph()
+    G.add_nodes_from(nodes)
+    colors = dict(zip(nodes, get_hex_colors(len(nodes))))
+    nx.set_node_attributes(G, colors, 'color')
+    for s,t,label in rels:
+        G.add_edge(s,t,label=label)
+        
+    return G
+            
 def networkx_to_cytoscape(graph):
     cy_graph = json_graph.cytoscape_data(graph)
     cy_nodes = cy_graph['elements']['nodes']
@@ -83,6 +115,13 @@ def networkx_to_cytoscape(graph):
 
     return cy_elements, mouseover_node
 
+def networkx_to_gml(graph, path):
+    nx.write_gml(graph, path)
+
+def json_network_to_gml(graph_json, path):
+    graph = json_network_to_networkx(graph_json)
+    nx.write_gml(graph, path)
+
 def json_network_to_networkx(graph_json):
     graph = json_graph.node_link_graph(graph_json)
 

src/analytics_core/viz/viz.py

@@ -1074,7 +1074,7 @@ def get_network(data, identifier, args):
     Example::
 
         result = get_network(data, identifier='network', args={'source':'node1', 'target':'node2', 'cutoff':0.5, 'cutoff_abs':True, 'values':'weight', \
-                            'node_size':'ev_centrality', 'title':'Network Figure', 'color_weight': True})
+                            'node_size':'degree', 'title':'Network Figure', 'color_weight': True})
     """
     net = None
     if 'cutoff_abs' not in args:
@@ -1094,9 +1094,11 @@ def get_network(data, identifier, args):
             data[args["values"]] = 1 
 
         data = data.rename(index=str, columns={args['values']: "width"})
-        data['edge_width'] = data['width'].apply(np.abs)
-        min_edge_value = data['edge_width'].min()
-        max_edge_value = data['edge_width'].max()
+        data = data.fillna('null')
+        data.columns = [c.replace('_', '') for c in data.columns]
+        data['edgewidth'] = data['width'].apply(np.abs)
+        min_edge_value = data['edgewidth'].min()
+        max_edge_value = data['edgewidth'].max()
         graph = nx.from_pandas_edgelist(data, args['source'], args['target'], edge_attr=True)
 
         degrees = dict(graph.degree())
@@ -1154,7 +1156,7 @@ def get_network(data, identifier, args):
         edges_fig_table = get_table(edges_table, identifier=identifier+"_edges_table", title=args['title']+" edges table")
 
         stylesheet, layout = get_network_style(colors, args['color_weight'])
-        stylesheet.append({'selector':'edge','style':{'width':'mapData(edge_width,'+ str(min_edge_value) +','+ str(max_edge_value) +', .5, 8)'}})
+        stylesheet.append({'selector':'edge','style':{'width':'mapData(edgewidth,'+ str(min_edge_value) +','+ str(max_edge_value) +', .5, 8)'}})
         if min_node_size > 0 and max_node_size >0:
             mapper = 'mapData(radius,'+ str(min_node_size) +','+ str(max_node_size) +', 15, 50)'
             stylesheet.append({'selector':'node','style':{'width':mapper, 'height':mapper}})
@@ -1181,15 +1183,22 @@ def get_network_style(node_colors, color_edges):
     '''
 
     color_selector = "{'selector': '[name = \"KEY\"]', 'style': {'background-color': 'VALUE'}}"
-    stylesheet=[{'selector': 'node', 'style': {'label': 'data(name)'}},
-                {'selector':'edge','style':{'curve-style': 'bezier'}}]
+    stylesheet=[{'selector': 'node', 'style': {'label': 'data(name)', 
+                                               'text-valign': 'center', 
+                                               'text-halign': 'center', 
+                                               'border-color':'gray', 
+                                               'border-width': '1px',
+                                               'font-size': '12',
+                                               'opacity':0.75}},
+                {'selector':'edge','style':{'label':'data(label)',
+                                            'curve-style': 'bezier',
+                                            'opacity':0.7,
+                                            'font-size': '4'}}]
 
     layout = {'name': 'cose',
                 'idealEdgeLength': 100,
                 'nodeOverlap': 20,
                 'refresh': 20,
-                #'fit': True,
-                #'padding': 30,
                 'randomize': False,
                 'componentSpacing': 100,
                 'nodeRepulsion': 400000,
@@ -1970,14 +1979,20 @@ def get_cytoscape_network(net, identifier, args):
         * **layout** (dict) -- specifies how the nodes should be positioned on the screen.
     :return: network figure within <div id="_dash-app-content">.
     """
+    height = '700px'    
+    width = '100%'
+    if 'height' in args:
+        height = args['height']
+    if 'width' in args:
+        width = args['width']
     cytonet = html.Div([html.H2(args['title']), cyto.Cytoscape(id=identifier,
                                     stylesheet=args['stylesheet'],
                                     elements=net,
                                     layout=args['layout'],
                                     minZoom = 0.2,
                                     maxZoom = 1.5,
                                     #mouseoverNodeData=args['mouseover_node'],
-                                    style={'width': '100%', 'height': '700px'}
+                                    style={'width': width, 'height': height}
                                     )
                     ])
 

src/graphdb_builder/builder/builder_config.yml

@@ -12,13 +12,16 @@ experimentsDirectory: '../../../data/imports/experiments'
 databasesDirectory: '../../../data/imports/databases'
 #ontologies import diretory
 ontologiesDirectory: '../../../data/imports/ontologies'
-#users import directory
-usersDirectory: '../../../data/imports/users'
+#users directory
+usersDirectory: '../../../data/users'
+usersImportDirectory: '../../../data/imports/users'
 
 download: True
 
 cypher_queries_file: 'cypher.yml'
 
+usersFile: 'CKG_users.xlsx'
+
 statsFile: 'stats.hdf'
 statsCols: 
     - 'date'
@@ -52,14 +55,14 @@ graph:
     - functional_regions
     - side effects
     - metabolite
+    - protein_structure
     - food
     - gwas
     - pathway
     - published
     - user
     - project
     - experiment
-    - protein_structure
 
 #Imports 
 ontology_entities: 

src/graphdb_builder/builder/cypher.yml

@@ -654,6 +654,13 @@ IMPORT_DATASETS:
         'name': 'import clinical data'
         'description': 'Loads into the database all the clinical variables measured in a project'
         'query': >
+            USING PERIODIC COMMIT 10000
+            LOAD CSV WITH HEADERS FROM "file:///IMPORTDIR/PROJECTID_subject_had_intervention.tsv" AS line
+            FIELDTERMINATOR '\t'
+            MATCH (subject:Subject {id:line.START_ID})
+            MATCH (intervention:Clinical_variable {id:line.END_ID})
+            MERGE (subject)-[r:HAD_INTERVENTION{type:line.type, in_combination:line.in_combination, response:line.response}]->(intervention)
+            RETURN COUNT(r) AS interventions;
             USING PERIODIC COMMIT 10000 
             LOAD CSV WITH HEADERS FROM "file:///IMPORTDIR/PROJECTID_clinical_state.tsv" AS line 
             FIELDTERMINATOR '\t'

src/graphdb_connector/query_utils.py

@@ -34,10 +34,10 @@ def find_queries_involving_nodes(queries, nodes, print_pretty=False):
             print("Query Name: ", query["name"])
             print("Description: ", query["description"])
             print("Involves nodes:", ",".join(query['involved_nodes']))
-            print("Involves relationships:", ",".join(query['involved_nodes']))
+            print("Involves relationships:", ",".join(query['involved_rels']))
             print("Query:\n", query["query"])
             if "example" in query:
-                print("How to use it:\n", "\n".join(query["example"]))
+                print("How to use it:\n", "\n".join(query["example"]), "\n")
 
     return valid_queries