Merge pull request #23 from microsoft/m-kovalsky/tomfixes

fixed add_time_intel in tom, added is_calculated_table, ran black
microsoft · Jul 8, 2024 · 1f13340 · 1f13340
2 parents fb55954 + bc7ad4c
commit 1f13340
Show file tree

Hide file tree

Showing 40 changed files with 918 additions and 405 deletions.
diff --git a/README.md b/README.md
@@ -1,22 +1,22 @@
-# [semantic-link-labs](https://semantic-link-labs.readthedocs.io/en/0.5.0/)
+# Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
 [![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.5.0&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
-All functions in this library are documented [here](https://semantic-link-labs.readthedocs.io/en/0.5.0/)!
+---
+[Read the documentation on ReadTheDocs!](https://semantic-link-labs.readthedocs.io/en/stable/)
+---
 
-This is a python library intended to be used in [Microsoft Fabric notebooks](https://learn.microsoft.com/fabric/data-engineering/how-to-use-notebook). This library was originally intended to contain functions used for [migrating semantic models to Direct Lake mode](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration). However, it quickly became apparent that functions within such a library could support many other useful activities in the realm of semantic models, reports, lakehouses and really anything Fabric-related. As such, this library contains a variety of functions ranging from running [Vertipaq Analyzer](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.html#sempy_labs.import_vertipaq_analyzer) or the [Best Practice Analyzer](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.html#sempy_labs.run_model_bpa) against a semantic model to seeing if any [lakehouse tables hit Direct Lake guardrails](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.lakehouse.html#sempy_labs.lakehouse.get_lakehouse_tables) or accessing the [Tabular Object Model](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.tom.html) and more!
+This is a python library intended to be used in [Microsoft Fabric notebooks](https://learn.microsoft.com/fabric/data-engineering/how-to-use-notebook). This library was originally intended to solely contain functions used for [migrating semantic models to Direct Lake mode](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration). However, it quickly became apparent that functions within such a library could support many other useful activities in the realm of semantic models, reports, lakehouses and really anything Fabric-related. As such, this library contains a variety of functions ranging from running [Vertipaq Analyzer](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.import_vertipaq_analyzer) or the [Best Practice Analyzer](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.run_model_bpa) against a semantic model to seeing if any [lakehouse tables hit Direct Lake guardrails](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.lakehouse.html#sempy_labs.lakehouse.get_lakehouse_tables) or accessing the [Tabular Object Model](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.tom.html) and more!
 
 Instructions for migrating import/DirectQuery semantic models to Direct Lake mode can be found [here](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration).
 
 If you encounter any issues, please [raise a bug](https://github.com/microsoft/semantic-link-labs/issues/new?assignees=&labels=&projects=&template=bug_report.md&title=).
 
 If you have ideas for new features/functions, please [request a feature](https://github.com/microsoft/semantic-link-labs/issues/new?assignees=&labels=&projects=&template=feature_request.md&title=).
 
-## [Function documentation](https://semantic-link-labs.readthedocs.io/en/0.5.0/)
-
 ## Install the library in a Fabric notebook
 ```python
 %pip install semantic-link-labs

diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py
@@ -66,6 +66,7 @@
     resolve_report_name,
     #  language_validate
 )
+
 #  from sempy_labs._model_auto_build import model_auto_build
 from sempy_labs._model_bpa import model_bpa_rules, run_model_bpa
 from sempy_labs._model_dependencies import (
@@ -125,7 +126,7 @@
     #'list_sqlendpoints',
     #'list_tables',
     "list_warehouses",
-    'list_workspace_role_assignments',
+    "list_workspace_role_assignments",
     "create_warehouse",
     "update_item",
     "create_abfss_path",
@@ -141,20 +142,20 @@
     "resolve_report_id",
     "resolve_report_name",
     #'language_validate',
-    #"model_auto_build",
+    # "model_auto_build",
     "model_bpa_rules",
     "run_model_bpa",
     "measure_dependency_tree",
     "get_measure_dependencies",
     "get_model_calc_dependencies",
     "export_model_to_onelake",
-    'qso_sync',
-    'qso_sync_status',
-    'set_qso',
-    'list_qso_settings',
-    'disable_qso',
-    'set_semantic_model_storage_format',
-    'set_workspace_default_storage_format',
+    "qso_sync",
+    "qso_sync_status",
+    "set_qso",
+    "list_qso_settings",
+    "disable_qso",
+    "set_semantic_model_storage_format",
+    "set_workspace_default_storage_format",
     "refresh_semantic_model",
     "cancel_dataset_refresh",
     "translate_semantic_model",
@@ -174,5 +175,5 @@
     "delete_user_from_workspace",
     "update_workspace_user",
     "list_workspace_users",
-    "assign_workspace_to_dataflow_storage"
+    "assign_workspace_to_dataflow_storage",
 ]
diff --git a/src/sempy_labs/_ai.py b/src/sempy_labs/_ai.py
@@ -79,7 +79,9 @@ def generate_measure_descriptions(
 
     validModels = ["gpt-35-turbo", "gpt-35-turbo-16k", "gpt-4"]
     if gpt_model not in validModels:
-        raise ValueError(f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}.")
+        raise ValueError(
+            f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}."
+        )
 
     dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
 
@@ -114,8 +116,7 @@ def generate_measure_descriptions(
     )
 
     # Update the model to use the new descriptions
-    #with connect_semantic_model(dataset=dataset, workspace=workspace, readonly=False) as tom:
-
+    # with connect_semantic_model(dataset=dataset, workspace=workspace, readonly=False) as tom:
 
     # for t in m.Tables:
     # tName = t.Name
@@ -171,33 +172,43 @@ def generate_aggs(
     numericTypes = ["Int64", "Double", "Decimal"]
 
     if any(value not in aggTypes for value in columns.values()):
-        raise ValueError(f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}.")
+        raise ValueError(
+            f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
+        )
 
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
     dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
     dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
     if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
-        raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models.")
-
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
+        )
+
     dfC_filtT = dfC[dfC["Table Name"] == table_name]
 
     if len(dfC_filtT) == 0:
-        raise ValueError(f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
 
     dfC_filt = dfC[
         (dfC["Table Name"] == table_name) & (dfC["Column Name"].isin(columnValues))
     ]
 
     if len(columns) != len(dfC_filt):
-        raise ValueError(f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
 
     # Check if doing sum/count/min/max etc. on a non-number column
     for col, agg in columns.items():
         dfC_col = dfC_filt[dfC_filt["Column Name"] == col]
         dataType = dfC_col["Data Type"].iloc[0]
         if agg in aggTypesAggregate and dataType not in numericTypes:
-            raise ValueError(f"{icons.red_dot} The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types.")            
+            raise ValueError(
+                f"{icons.red_dot} The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types."
+            )
 
     # Create/update lakehouse delta agg table
     aggSuffix = "_agg"
@@ -213,7 +224,9 @@ def generate_aggs(
     dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
 
     if len(dfI_filt) == 0:
-        raise ValueError(f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter.")
+        raise ValueError(
+            f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
+        )
 
     lakehouseName = dfI_filt["Display Name"].iloc[0]
     lakehouse_id = resolve_lakehouse_id(
@@ -328,7 +341,9 @@ def generate_aggs(
             col.DataType = System.Enum.Parse(TOM.DataType, dType)
 
             m.Tables[aggTableName].Columns.Add(col)
-            print(f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added.")
+            print(
+                f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added."
+            )
 
     # Create relationships
     relMap = {"m": "Many", "1": "One", "0": "None"}

diff --git a/src/sempy_labs/_connections.py b/src/sempy_labs/_connections.py
@@ -64,13 +64,19 @@ def create_connection_cloud(
             "Connection Id": o.get("id"),
             "Connection Name": o.get("name"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
@@ -140,13 +146,19 @@ def create_connection_on_prem(
             "Connection Name": o.get("name"),
             "Gateway ID": o.get("gatewayId"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
@@ -218,13 +230,19 @@ def create_connection_vnet(
             "Connection Name": o.get("name"),
             "Gateway ID": o.get("gatewayId"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 

diff --git a/src/sempy_labs/_dax.py b/src/sempy_labs/_dax.py
@@ -50,7 +50,7 @@ def evaluate_dax_impersonation(
 
     request_body = {
         "queries": [{"query": dax_query}],
-        "impersonatedUserName": user_name
+        "impersonatedUserName": user_name,
     }
 
     client = fabric.PowerBIRestClient()

diff --git a/src/sempy_labs/_generate_semantic_model.py b/src/sempy_labs/_generate_semantic_model.py
@@ -37,7 +37,9 @@ def create_blank_semantic_model(
     min_compat = 1500
 
     if compatibility_level < min_compat:
-        raise ValueError(f"{icons.red_dot} Compatiblity level must be at least {min_compat}.")
+        raise ValueError(
+            f"{icons.red_dot} Compatiblity level must be at least {min_compat}."
+        )
 
     tmsl = f"""
   {{
@@ -90,7 +92,9 @@ def create_semantic_model_from_bim(
     dfI_filt = dfI[(dfI["Display Name"] == dataset)]
 
     if len(dfI_filt) > 0:
-        raise ValueError(f"{icons.red_dot} '{dataset}' already exists as a semantic model in the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} '{dataset}' already exists as a semantic model in the '{workspace}' workspace."
+        )
 
     client = fabric.FabricRestClient()
     defPBIDataset = {"version": "1.0", "settings": {}}

diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py
@@ -200,7 +200,9 @@ def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None):
     return obj
 
 
-def resolve_lakehouse_name(lakehouse_id: Optional[UUID] = None, workspace: Optional[str] = None):
+def resolve_lakehouse_name(
+    lakehouse_id: Optional[UUID] = None, workspace: Optional[str] = None
+):
     """
     Obtains the name of the Fabric lakehouse.
 
@@ -223,7 +225,7 @@ def resolve_lakehouse_name(lakehouse_id: Optional[UUID] = None, workspace: Optio
     if workspace is None:
         workspace_id = fabric.get_workspace_id()
         workspace = fabric.resolve_workspace_name(workspace_id)
-    
+
     if lakehouse_id is None:
         lakehouse_id = fabric.get_lakehouse_id()
 
@@ -420,10 +422,14 @@ def save_as_delta_table(
     write_mode = write_mode.lower()
 
     if write_mode not in writeModes:
-        raise ValueError(f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {writeModes}.")
+        raise ValueError(
+            f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {writeModes}."
+        )
 
     if " " in delta_table_name:
-        raise ValueError(f"{icons.red_dot} Invalid 'delta_table_name'. Delta tables in the lakehouse cannot have spaces in their names.")
+        raise ValueError(
+            f"{icons.red_dot} Invalid 'delta_table_name'. Delta tables in the lakehouse cannot have spaces in their names."
+        )
 
     dataframe.columns = dataframe.columns.str.replace(" ", "_")
 
@@ -470,7 +476,9 @@ def language_validate(language: str):
     elif len(df_filt2) == 1:
         lang = df_filt2["Language"].iloc[0]
     else:
-        raise ValueError(f"{icons.red_dot} The '{language}' language is not a valid language code. Please refer to this link for a list of valid language codes: {url}.")
+        raise ValueError(
+            f"{icons.red_dot} The '{language}' language is not a valid language code. Please refer to this link for a list of valid language codes: {url}."
+        )
 
     return lang