Merge pull request #114 from dengzq1234/desktop_v2

Desktop v2

treeprofiler/app.py

@@ -13,7 +13,7 @@
 
 
 from ete4 import Tree
-from treeprofiler.tree_annotate import run_tree_annotate, parse_csv, name_nodes  # or other functions you need
+from treeprofiler.tree_annotate import run_tree_annotate, run_array_annotate, parse_csv, parse_tsv_to_array, name_nodes  # or other functions you need
 from treeprofiler import tree_plot
 from treeprofiler.src import utils
 from treeprofiler import layouts
@@ -175,14 +175,20 @@ def do_upload():
     job_status[treename] = "running"
 
     # Collect all form data
-    
+
     job_args = {
         "treename": treename,
         "tree_data": request.forms.get('tree'),
         "treeparser": request.forms.get('treeparser'),
         "is_annotated_tree": request.forms.get('isAnnotatedTree') == 'true',
+
         "metadata": request.forms.get('metadata'),
         "separator": request.forms.get('separator'),
+
+        "matrix": request.forms.get('matrix'),
+        "matrix_name": request.forms.get('matrixFiles'),
+        "matrix_separator": request.forms.get('matrixSeparator'),
+
         "text_prop": request.forms.getlist('text_prop[]'),
         "num_prop": request.forms.getlist('num_prop[]'),
         "bool_prop": request.forms.getlist('bool_prop[]'),
@@ -243,6 +249,11 @@ def process_upload_job(job_args):
     metadata_file_paths = uploaded_chunks.get(treename, {}).get("metadata")
     metadata_file_list = list(metadata_file_paths.values())
 
+    matrix_file_paths = uploaded_chunks.get(treename, {}).get("matrix")
+
+    matrix_file_list = list(matrix_file_paths.values())
+    matrix_separator = "\t" if job_args.get("matrix_separator") == "<tab>" else job_args.get("matrix_separator", ",")
+
     alignment_file_path = uploaded_chunks.get(treename, {}).get("alignment")
     pfam_file_path = uploaded_chunks.get(treename, {}).get("pfam")
 
@@ -350,6 +361,7 @@ def process_upload_job(job_args):
             "multiple_text_prop": job_args.get("multiple_text_prop")
         }
 
+
         # Taxonomic annotation options
         taxonomic_options = {}
         if job_args.get("taxon_column"):
@@ -421,17 +433,43 @@ def process_upload_job(job_args):
             threads=threads
         )
 
+        # Process Matrix
+        node_props_array = []
+        if matrix_file_list:
+            tmp2filename = {os.path.basename(value): key for key, value in matrix_file_paths.items()}
+            array_dict = parse_tsv_to_array(matrix_file_list, delimiter=matrix_separator)
+            filename2array = {}
+
+            for key, value in array_dict.items():
+                filename2array[tmp2filename[key]] = value
+
+            annotated_tree = run_array_annotate(annotated_tree, filename2array, column2method=column2method)
+            # update prop2type
+            for filename in filename2array.keys():
+                prop2type[filename] = list
+                prop2type[utils.add_suffix(filename, 'avg')] = list
+                prop2type[utils.add_suffix(filename, 'max')] = list
+                prop2type[utils.add_suffix(filename, 'min')] = list
+                prop2type[utils.add_suffix(filename, 'sum')] = list
+                prop2type[utils.add_suffix(filename, 'std')] = list
+
+            node_props_array = list(filename2array.keys())
+
         if job_args.get("taxon_column"):
             rank_list = sorted(list(set(utils.tree_prop_array(annotated_tree, 'rank'))))
         else:
             rank_list = []
 
         # Post-processing of annotated tree properties
         list_keys = [key for key, value in prop2type.items() if value == list]
-        for node in annotated_tree.leaves():
+
+        list_sep = '||'
+        for node in annotated_tree.traverse():
             for key in list_keys:
                 if node.props.get(key):
-                    node.add_prop(key, '||'.join(node.props[key]))
+                    cont2str = list(map(str, node.props.get(key)))
+                    list2str = list_sep.join(cont2str)
+                    node.add_prop(key, list2str)
 
         # Name the nodes
         annotated_tree = name_nodes(annotated_tree)
@@ -443,6 +481,8 @@ def process_upload_job(job_args):
 
         # Add node properties for display
         node_props = metadata_options.get('node_props', [])
+        if node_props_array:
+            node_props.extend(node_props_array)
 
         # check if tree has support values
         sample_node = annotated_tree.children[0]
@@ -498,7 +538,7 @@ def upload_tree():
 @app.route('/upload_chunk', method='POST')
 def upload_chunk():
     # Determine the file type based on the request
-    chunk = request.files.get('treeFile') or request.files.get('metadataFile') or request.files.get('alignmentFile') or request.files.get('pfamFile')
+    chunk = request.files.get('treeFile') or request.files.get('metadataFile') or request.files.get('alignmentFile') or request.files.get('pfamFile') or request.files.get('matrixFile')
     chunk_index = int(request.forms.get("chunkIndex"))
     total_chunks = int(request.forms.get("totalChunks"))
     treename = request.forms.get("treename")
@@ -509,6 +549,8 @@ def upload_chunk():
         file_type = "tree"
     elif 'metadataFile' in request.files:
         file_type = "metadata"
+    elif 'matrixFile' in request.files:
+        file_type = "matrix"
     elif 'alignmentFile' in request.files:
         file_type = "alignment"
     elif 'pfamFile' in request.files:
@@ -518,10 +560,16 @@ def upload_chunk():
 
     # Initialize storage
     if treename not in uploaded_chunks:
-        uploaded_chunks[treename] = {"tree": {}, "metadata": {}, "alignment": {}, "pfam": {}}
+        uploaded_chunks[treename] = {
+            "tree": {}, 
+            "metadata": {}, 
+            "matrix": {},
+            "alignment": {}, 
+            "pfam": {}
+        }
 
     # Handle metadata specifically for multiple files
-    if file_type == "metadata":
+    if file_type == "metadata" or file_type == "matrix":
         if file_id not in uploaded_chunks[treename][file_type]:
             uploaded_chunks[treename][file_type][file_id] = []
         uploaded_chunks[treename][file_type][file_id].append((chunk_index, chunk.file.read()))
@@ -598,15 +646,16 @@ def explore_tree(treename):
 
     if current_layouts:
         layout_manager = {layout.name: layout for layout in current_layouts}
-    current_props = sorted(list(tree_info['prop2type'].keys()))
 
+    prop2type = tree_info['prop2type']
+    current_props = sorted(list(tree_info['prop2type'].keys()))
+
     if tree_info['updated_tree']:
         t = Tree(tree_info['updated_tree'])
     else:
         t = Tree(tree_info['annotated_tree'])
 
 
-
     # Default configuration settings
     default_configs = {
         "level": 1,
@@ -659,14 +708,13 @@ def explore_tree(treename):
                         # prune tree by condition 
                         query_box = layer.get('query', '')
                         query_strings = convert_query_string(query_box)
-                        prop2type = tree_info['prop2type']
                         t = utils.conditional_prune(t, query_strings, prop2type)
                         tree_info['updated_tree'] = t.write(props=current_props, format_root_node=True)
 
                     # Process each layer individually without altering its structure
                     current_layouts, current_props, level, color_config = process_layer(
                         t, layer, tree_info, current_layouts, current_props, level,
-                        column_width, padding_x, padding_y, color_config, internal_num_rep, default_paired_color
+                        column_width, padding_x, padding_y, color_config, internal_num_rep, default_paired_color, 
                     )
 
 
@@ -1080,7 +1128,7 @@ def explore_tree(treename):
                                 matrix, minval, maxval, value2color, results_list, list_props, single_props = tree_plot.numerical2matrix(t, 
                                 selected_props, count_negative=True, 
                                 internal_num_rep=layout_meta['config']['internal_num_rep'], 
-                                color_config=color_config, norm_method='min-max')
+                                color_config=color_config, norm_method='min-max', prop2type=tree_info['prop2type'])
 
                                 # TODO add list_props at this moment
                                 if list_props:
@@ -1465,7 +1513,7 @@ def process_layer(t, layer, tree_info, current_layouts, current_props, level, co
             color_config[prop]['detail2color']['color_min'] = (color_min, minval)
         matrix, minval, maxval, value2color, results_list, list_props, single_props = tree_plot.numerical2matrix(t, 
         selected_props, count_negative=True, internal_num_rep=internal_num_rep, 
-        color_config=color_config, norm_method='min-max')
+        color_config=color_config, norm_method='min-max', prop2type=prop2type)
         if list_props:
             index_map = {value: idx for idx, value in enumerate(selected_props)}
             sorted_list_props = sorted(list_props, key=lambda x: index_map[x])

treeprofiler/src/utils.py

@@ -430,7 +430,7 @@ def conditional_prune(tree, conditions_input, prop2type):
 
 def tree_prop_array(node, prop, leaf_only=False, numeric=False, list_type=False):
     array = []
-    sep = '||'
+    list_sep = '||'
 
     # Decide whether to traverse all nodes or only leaves
     nodes = node.leaves() if leaf_only else node.traverse()
@@ -442,7 +442,7 @@ def tree_prop_array(node, prop, leaf_only=False, numeric=False, list_type=False)
 
             # Handle list-type property values
             if list_type:
-                prop_value = prop_value.split(sep)
+                prop_value = prop_value.split(list_sep)
                 if numeric:
                     try:
                         # Convert elements to floats, replace empty with NaN

treeprofiler/tree_annotate.py

@@ -678,13 +678,13 @@ def run_tree_annotate(tree, input_annotated_tree=False,
 def run_array_annotate(tree, array_dict, num_stat='none', column2method={}):
     matrix_props = list(array_dict.keys())
     # annotate to the leaves
+    start = time.time()
     for node in tree.traverse():
         if node.is_leaf:
             for filename, array in array_dict.items():
                 if array.get(node.name):
                     node.add_prop(filename, array.get(node.name))
 
-
     # merge annotations to internal nodes
     for node in tree.traverse():
         if not node.is_leaf:
@@ -700,6 +700,8 @@ def run_array_annotate(tree, array_dict, num_stat='none', column2method={}):
                     for stat, value in stats.items():
                         node.add_prop(utils.add_suffix(prop, stat), value.tolist())
                         #prop2type[utils.add_suffix(prop, stat)] = float
+    end = time.time()
+    logger.info(f'Time for run_array_annotate to run: {end - start}')
     return tree
 
 
@@ -892,6 +894,9 @@ def run(args):
 
     if args.data_matrix:
         annotated_tree = run_array_annotate(annotated_tree, array_dict, num_stat=args.num_stat, column2method=column2method)
+        # update prop2type
+        for filename in array_dict.keys():
+            prop2type[filename] = list
 
     if args.outdir:
         base=os.path.splitext(os.path.basename(args.tree))[0]
@@ -923,22 +928,26 @@ def run(args):
         ### out newick
         ## need to correct wrong symbols in the newick tree, such as ',' -> '||'
         # Find all keys where the value is of type list
+
         list_keys = [key for key, value in prop2type.items() if value == list]
         # Replace all commas in the tree with '||'
         list_sep = '||'
         for node in annotated_tree.leaves():
             for key in list_keys:
                 if node.props.get(key):
-                    list2str = list_sep.join(node.props.get(key))
+                    cont2str = list(map(str, node.props.get(key)))
+                    list2str = list_sep.join(cont2str)
                     node.add_prop(key, list2str)
+
 
         avail_props = list(prop2type.keys())
 
         #del avail_props[avail_props.index('name')]
         del avail_props[avail_props.index('dist')]
+        del avail_props[avail_props.index('name')]
         if 'support' in avail_props:
             del avail_props[avail_props.index('support')]
-    
+
         annotated_tree.write(outfile=os.path.join(args.outdir, out_newick), props=avail_props, 
                     parser=utils.get_internal_parser(args.internal), format_root_node=True)
 

treeprofiler/tree_plot.py

@@ -699,7 +699,7 @@ def run(args):
             # if is list, it should provide more than one matrix
             matrix, minval, maxval, value2color, results_list, list_props, single_props = numerical2matrix(tree, 
                 numerical_props, count_negative=True, internal_num_rep=internal_num_rep, 
-                color_config=color_config, norm_method='min-max')
+                color_config=color_config, norm_method='min-max', prop2type=prop2type, eteformat_flag=eteformat_flag)
 
             if list_props:
                 index_map = {value: idx for idx, value in enumerate(numerical_props)}
@@ -1817,7 +1817,7 @@ def categorical2matrix(tree, profiling_props, dtype=str, color_config=None):
 
     return leaf2matrix, value2color
 
-def numerical2matrix(tree, profiling_props, count_negative=True, internal_num_rep=None, color_config=None, norm_method='min-max'):
+def numerical2matrix(tree, profiling_props, count_negative=True, internal_num_rep=None, color_config=None, norm_method='min-max', prop2type=None, eteformat_flag=False):
     """
     Input:
     tree: A tree structure with nodes, each having properties.
@@ -1994,28 +1994,38 @@ def process_color_configuration(node2matrix, profiling_props=None):
 
     single_props = set()
     list_props = set()
-
+    list_sep = '||'
+
     for node in tree.traverse():
         node2matrix_single[node.name] = []
         for profiling_prop in profiling_props:
+            data_type = prop2type.get(profiling_prop, None)
             prop_value = node.props.get(profiling_prop)
+
             if prop_value is not None:
-                if isinstance(prop_value, list):
+                if isinstance(prop_value, list) or data_type == list:
                     list_props.add(profiling_prop)
+
+                    if not eteformat_flag:
+                        prop_value = prop_value.split(list_sep)
+
                     prop_value = list(map(float, prop_value))
                     if node.name not in node2matrix_list[profiling_prop]:
                         node2matrix_list[profiling_prop][node.name] = []
                     node2matrix_list[profiling_prop][node.name] = prop_value
                 else:
                     single_props.add(profiling_prop)
                     node2matrix_single[node.name].append(float(prop_value))
+
             else:
                 if internal_num_rep != 'none':
                     representative_prop = utils.add_suffix(profiling_prop, internal_num_rep)
                     prop_value = node.props.get(representative_prop)
                     if prop_value is not None:
-                        if isinstance(prop_value, list):
+                        if isinstance(prop_value, list) or prop2type.get(representative_prop) == list:
                             list_props.add(profiling_prop)
+                            if not eteformat_flag:
+                                prop_value = prop_value.split(list_sep)
                             prop_value = list(map(float, prop_value))
                             if node.name not in node2matrix_list[profiling_prop]:
                                 node2matrix_list[profiling_prop][node.name] = []

treeprofiler/views/explore_tree_v3.html

@@ -88,7 +88,7 @@
     </style>
 </head>
 <body>
-<div class="container-fluid mt-4">
+<div class="container-fluid mt-0">
     <ul class="nav nav-tabs" id="controlPanelTabs" role="tablist">
         <li class="nav-item">
             <button class="nav-link active" id="properties-tab" data-bs-toggle="tab" data-bs-target="#properties" type="button" role="tab" aria-controls="properties" aria-selected="true">Metadata-Based Layouts</button>