Merge pull request #8 from rfdearborn/rfdearborn/some-cleanup

get_path_or_empty cleanup

Author: rfdearborn

dbt_docs_to_notion.py

@@ -31,46 +31,31 @@ def make_request(endpoint, querystring='', method='GET', **request_kwargs):
   return resp.json()
 
 
-def get_path_or_empty(parent_object, path_array, zero_value=''):
+def get_paths_or_empty(parent_object, paths_array, zero_value=''):
+  """Used for catalog_nodes accesses, since structure is variable"""
+  for path in paths_array:
     obj = parent_object
-    for el in path_array:
-        if el not in obj:
-            return zero_value
-        obj = obj[el]
+    for el in path:
+      if el not in obj:
+        obj = zero_value
+        break
+      obj = obj[el]
+    if obj != zero_value:
+      return obj
 
-    return obj
-
-
-def get_num_rows(catalog_nodes, model_name):
-    zero_value = NUMERIC_ZERO_VALUE
-    keys = ['num_rows', 'row_count']
-    for key in keys:
-        num_rows = get_path_or_empty(catalog_nodes, [model_name, 'stats', key, 'value'], NUMERIC_ZERO_VALUE)
-        if num_rows != NUMERIC_ZERO_VALUE:
-            return num_rows
-
-    return NUMERIC_ZERO_VALUE
-
-
-def get_bytes(catalog_nodes, model_name):
-    zero_value = NUMERIC_ZERO_VALUE
-    keys = ['num_bytes', 'bytes']
-    for key in keys:
-        num_rows = get_path_or_empty(catalog_nodes, [model_name, 'stats', key, 'value'], NUMERIC_ZERO_VALUE)
-        if num_rows != NUMERIC_ZERO_VALUE:
-            return num_rows
-
-    return NUMERIC_ZERO_VALUE
+  return zero_value
 
 
 def get_owner(data, catalog_nodes, model_name):
-    # Check for an owner field explicitly named in the DBT Config
-    # If none present, fall back to database table owner
-    owner = get_path_or_empty(data, ['config', 'meta', 'owner'], None)
-    if owner != None:
-        return owner
+  """
+  Check for an owner field explicitly named in the DBT Config
+  If none present, fall back to database table owner
+  """
+  owner = get_paths_or_empty(data, [['config', 'meta', 'owner']], None)
+  if owner is not None:
+    return owner
 
-    return get_path_or_empty(catalog_nodes, [model_name, 'metadata', 'owner'], '')
+  return get_paths_or_empty(catalog_nodes, [[model_name, 'metadata', 'owner']], '')
 
 
 def main():
@@ -153,7 +138,7 @@ def main():
       }
     }
 
-    print('created creating database')
+    print('creating database')
     database_creation_resp = make_request(
       endpoint='databases/',
       querystring='',
@@ -207,7 +192,11 @@ def main():
           }
         }
       ]
-      col_names_and_data = list(get_path_or_empty(catalog_nodes, [model_name, 'columns'], {}).items())
+      col_names_and_data = list(get_paths_or_empty(
+        catalog_nodes,
+        [[model_name, 'columns']],
+        {}
+      ).items())
       for (col_name, col_data) in col_names_and_data[:98]: # notion api limit is 100 table rows
         columns_table_children_obj.append(
           {
@@ -427,10 +416,20 @@ def main():
             ]
           },
           "Approx Rows": {
-            "number": get_num_rows(catalog_nodes, model_name)
+            "number": get_paths_or_empty(
+              catalog_nodes,
+              [[model_name, 'stats', 'num_rows', 'value'],
+               [model_name, 'stats', 'row_count', 'value']],
+              NUMERIC_ZERO_VALUE
+            )
           },
           "Approx GB": {
-            "number":get_bytes(catalog_nodes, model_name) /1e9
+            "number": get_paths_or_empty(
+              catalog_nodes,
+              [[model_name, 'stats', 'bytes', 'value'],
+               [model_name, 'stats', 'num_bytes', 'value']],
+              NUMERIC_ZERO_VALUE
+            ) / 1e9
           },
           "Depends On": {
             "rich_text": [