chore(column): remove columns which contain all None

uc-cdis · Apr 4, 2018 · 6b6ec5c · 6b6ec5c
1 parent 3779f8e
commit 6b6ec5c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/peregrine/utils/json2csv.py b/peregrine/utils/json2csv.py
@@ -66,7 +66,9 @@ def dicts2tsv(dict_list):
     header_set = set()
 
     for dict_row in dict_list:
-        header_set.update(dict_row.keys())
+        for key in dict_row.keys():
+            if (dict_row[key] is not None and dict_row[key] != []):
+                header_set.update([key])
 
     for h in header_set:
         words = h.split('-')

diff --git a/peregrine/utils/pybdbag.py b/peregrine/utils/pybdbag.py
@@ -32,7 +32,9 @@ def create_bdbag(bag_info, payload, max_row=1000):
         header_set = set()
 
         for dict_row in json_data:
-            header_set.update(dict_row.keys())
+            for key in dict_row.keys():
+                if (dict_row[key] is not None and dict_row[key] != []):
+                    header_set.update([key])
 
         with open(bag_path + '/data/' + node_name + '.tsv', 'w') as tsvfile:
             writer = csv.writer(tsvfile, delimiter='\t')
@@ -65,7 +67,6 @@ def create_bdbag(bag_info, payload, max_row=1000):
     shutil.rmtree(zip_dir)
     return zip_file_name
 
-
 def zipdir(path, ziph):
     length = len(path)
     # ziph is zipfile handle