Merge branch 'main' into m-kovalsky/listfunctionfixes

Author: m-kovalsky

README.md

@@ -1,22 +1,22 @@
-# [semantic-link-labs](https://semantic-link-labs.readthedocs.io/en/0.5.0/)
+# Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
 [![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.5.0&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
-All functions in this library are documented [here](https://semantic-link-labs.readthedocs.io/en/0.5.0/)!
+---
+[Read the documentation on ReadTheDocs!](https://semantic-link-labs.readthedocs.io/en/stable/)
+---
 
-This is a python library intended to be used in [Microsoft Fabric notebooks](https://learn.microsoft.com/fabric/data-engineering/how-to-use-notebook). This library was originally intended to contain functions used for [migrating semantic models to Direct Lake mode](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration). However, it quickly became apparent that functions within such a library could support many other useful activities in the realm of semantic models, reports, lakehouses and really anything Fabric-related. As such, this library contains a variety of functions ranging from running [Vertipaq Analyzer](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.html#sempy_labs.import_vertipaq_analyzer) or the [Best Practice Analyzer](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.html#sempy_labs.run_model_bpa) against a semantic model to seeing if any [lakehouse tables hit Direct Lake guardrails](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.lakehouse.html#sempy_labs.lakehouse.get_lakehouse_tables) or accessing the [Tabular Object Model](https://semantic-link-labs.readthedocs.io/en/0.5.0/sempy_labs.tom.html) and more!
+This is a python library intended to be used in [Microsoft Fabric notebooks](https://learn.microsoft.com/fabric/data-engineering/how-to-use-notebook). This library was originally intended to solely contain functions used for [migrating semantic models to Direct Lake mode](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration). However, it quickly became apparent that functions within such a library could support many other useful activities in the realm of semantic models, reports, lakehouses and really anything Fabric-related. As such, this library contains a variety of functions ranging from running [Vertipaq Analyzer](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.import_vertipaq_analyzer) or the [Best Practice Analyzer](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.run_model_bpa) against a semantic model to seeing if any [lakehouse tables hit Direct Lake guardrails](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.lakehouse.html#sempy_labs.lakehouse.get_lakehouse_tables) or accessing the [Tabular Object Model](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.tom.html) and more!
 
 Instructions for migrating import/DirectQuery semantic models to Direct Lake mode can be found [here](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct-lake-migration).
 
 If you encounter any issues, please [raise a bug](https://github.com/microsoft/semantic-link-labs/issues/new?assignees=&labels=&projects=&template=bug_report.md&title=).
 
 If you have ideas for new features/functions, please [request a feature](https://github.com/microsoft/semantic-link-labs/issues/new?assignees=&labels=&projects=&template=feature_request.md&title=).
 
-## [Function documentation](https://semantic-link-labs.readthedocs.io/en/0.5.0/)
-
 ## Install the library in a Fabric notebook
 ```python
 %pip install semantic-link-labs

notebooks/Tabular Object Model.ipynb

@@ -66,6 +66,7 @@
     resolve_report_name,
     #  language_validate
 )
+
 #  from sempy_labs._model_auto_build import model_auto_build
 from sempy_labs._model_bpa import model_bpa_rules, run_model_bpa
 from sempy_labs._model_dependencies import (
@@ -125,7 +126,7 @@
     #'list_sqlendpoints',
     #'list_tables',
     "list_warehouses",
-    'list_workspace_role_assignments',
+    "list_workspace_role_assignments",
     "create_warehouse",
     "update_item",
     "create_abfss_path",
@@ -141,20 +142,20 @@
     "resolve_report_id",
     "resolve_report_name",
     #'language_validate',
-    #"model_auto_build",
+    # "model_auto_build",
     "model_bpa_rules",
     "run_model_bpa",
     "measure_dependency_tree",
     "get_measure_dependencies",
     "get_model_calc_dependencies",
     "export_model_to_onelake",
-    'qso_sync',
-    'qso_sync_status',
-    'set_qso',
-    'list_qso_settings',
-    'disable_qso',
-    'set_semantic_model_storage_format',
-    'set_workspace_default_storage_format',
+    "qso_sync",
+    "qso_sync_status",
+    "set_qso",
+    "list_qso_settings",
+    "disable_qso",
+    "set_semantic_model_storage_format",
+    "set_workspace_default_storage_format",
     "refresh_semantic_model",
     "cancel_dataset_refresh",
     "translate_semantic_model",
@@ -174,5 +175,5 @@
     "delete_user_from_workspace",
     "update_workspace_user",
     "list_workspace_users",
-    "assign_workspace_to_dataflow_storage"
+    "assign_workspace_to_dataflow_storage",
 ]

src/sempy_labs/__init__.py

@@ -14,7 +14,6 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
     from ._model_bpa import run_model_bpa
     from .directlake._fallback import check_fallback_reason
     from ._helper_functions import format_dax_object_name
-    from sempy_labs.tom import connect_semantic_model
 
     modelBPA = run_model_bpa(
         dataset=dataset, workspace=workspace, return_dataframe=True
@@ -41,7 +40,8 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
 
         if len(fallback_filt) > 0:
             print(
-                f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
+                f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. "
+                "Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
             )
 
     # Potential model reduction estimate
@@ -79,7 +79,9 @@ def generate_measure_descriptions(
 
     validModels = ["gpt-35-turbo", "gpt-35-turbo-16k", "gpt-4"]
     if gpt_model not in validModels:
-        raise ValueError(f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}.")
+        raise ValueError(
+            f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}."
+        )
 
     dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
 
@@ -114,8 +116,7 @@ def generate_measure_descriptions(
     )
 
     # Update the model to use the new descriptions
-    #with connect_semantic_model(dataset=dataset, workspace=workspace, readonly=False) as tom:
-    
+    # with connect_semantic_model(dataset=dataset, workspace=workspace, readonly=False) as tom:
 
     # for t in m.Tables:
     # tName = t.Name
@@ -146,10 +147,10 @@ def generate_aggs(
     import System
 
     # columns = {
-    #'SalesAmount': 'Sum',
-    #'ProductKey': 'GroupBy',
-    #'OrderDateKey': 'GroupBy'
-    # }
+    # 'SalesAmount': 'Sum',
+    # 'ProductKey': 'GroupBy',
+    # 'OrderDateKey': 'GroupBy'
+    #  }
 
     if workspace is None:
         workspace_id = fabric.get_workspace_id()
@@ -171,33 +172,44 @@ def generate_aggs(
     numericTypes = ["Int64", "Double", "Decimal"]
 
     if any(value not in aggTypes for value in columns.values()):
-        raise ValueError(f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}.")
+        raise ValueError(
+            f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
+        )
 
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
     dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
     dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
     if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
-        raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models.")
-        
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
+        )
+
     dfC_filtT = dfC[dfC["Table Name"] == table_name]
 
     if len(dfC_filtT) == 0:
-        raise ValueError(f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
 
     dfC_filt = dfC[
         (dfC["Table Name"] == table_name) & (dfC["Column Name"].isin(columnValues))
     ]
 
     if len(columns) != len(dfC_filt):
-        raise ValueError(f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
 
     # Check if doing sum/count/min/max etc. on a non-number column
-    for col, agg in columns.items():
-        dfC_col = dfC_filt[dfC_filt["Column Name"] == col]
+    for cm, agg in columns.items():
+        dfC_col = dfC_filt[dfC_filt["Column Name"] == cm]
         dataType = dfC_col["Data Type"].iloc[0]
         if agg in aggTypesAggregate and dataType not in numericTypes:
-            raise ValueError(f"{icons.red_dot} The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types.")            
+            raise ValueError(
+                f"{icons.red_dot} The '{cm}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types"
+                f" can be aggregated as '{aggTypesAggregate}' aggregation types."
+            )
 
     # Create/update lakehouse delta agg table
     aggSuffix = "_agg"
@@ -213,7 +225,10 @@ def generate_aggs(
     dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
 
     if len(dfI_filt) == 0:
-        raise ValueError(f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter.")
+        raise ValueError(
+            f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in"
+            f" the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
+        )
 
     lakehouseName = dfI_filt["Display Name"].iloc[0]
     lakehouse_id = resolve_lakehouse_id(
@@ -223,8 +238,8 @@ def generate_aggs(
     # Generate SQL query
     query = "SELECT"
     groupBy = "\nGROUP BY"
-    for col, agg in columns.items():
-        colFilt = dfC_filt[dfC_filt["Column Name"] == col]
+    for cm, agg in columns.items():
+        colFilt = dfC_filt[dfC_filt["Column Name"] == cm]
         sourceCol = colFilt["Source"].iloc[0]
 
         if agg == "GroupBy":
@@ -328,7 +343,9 @@ def generate_aggs(
             col.DataType = System.Enum.Parse(TOM.DataType, dType)
 
             m.Tables[aggTableName].Columns.Add(col)
-            print(f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added.")
+            print(
+                f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added."
+            )
 
     # Create relationships
     relMap = {"m": "Many", "1": "One", "0": "None"}
@@ -367,22 +384,24 @@ def generate_aggs(
                 print(
                     f"{icons.green_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
                 )
-            except:
+            except Exception as e:
                 print(
                     f"{icons.red_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
                 )
+                print(f"Exception occured: {e}")
         elif toTable == table_name:
             try:
                 rel.ToColumn = m.Tables[aggTableName].Columns[toColumn]
                 m.Relationships.Add(rel)
                 print(
                     f"{icons.green_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
                 )
-            except:
+            except Exception as e:
                 print(
                     f"{icons.red_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
                 )
-    f"Relationship creation is complete."
+                print(f"Exception occured: {e}")
+    "Relationship creation is complete."
 
     # Create IF measure
     f"\n{icons.in_progress} Creating measure to check if the agg table can be used..."

src/sempy_labs/_ai.py

@@ -1,7 +1,6 @@
-import sempy
 import sempy.fabric as fabric
 from ._helper_functions import resolve_dataset_id
-from typing import List, Optional, Union
+from typing import Optional
 import sempy_labs._icons as icons
 
 
@@ -25,10 +24,10 @@ def clear_cache(dataset: str, workspace: Optional[str] = None):
     datasetID = resolve_dataset_id(dataset=dataset, workspace=workspace)
 
     xmla = f"""
-            <ClearCache xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">  
+            <ClearCache xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">
                 <Object>  
-                    <DatabaseID>{datasetID}</DatabaseID>  
-                </Object>  
+                    <DatabaseID>{datasetID}</DatabaseID>
+                </Object>
             </ClearCache>
             """
     fabric.execute_xmla(dataset=dataset, xmla_command=xmla, workspace=workspace)

src/sempy_labs/_clear_cache.py

@@ -1,7 +1,5 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
-from typing import List, Optional, Union
 import sempy_labs._icons as icons
 
 
@@ -56,21 +54,27 @@ def create_connection_cloud(
         },
     }
 
-    response = client.post(f"/v1/connections", json=request_body)
+    response = client.post("/v1/connections", json=request_body)
 
     if response.status_code == 200:
         o = response.json()
         new_data = {
             "Connection Id": o.get("id"),
             "Connection Name": o.get("name"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
@@ -131,7 +135,7 @@ def create_connection_on_prem(
         },
     }
 
-    response = client.post(f"/v1/connections", json=request_body)
+    response = client.post("/v1/connections", json=request_body)
 
     if response.status_code == 200:
         o = response.json()
@@ -140,13 +144,19 @@ def create_connection_on_prem(
             "Connection Name": o.get("name"),
             "Gateway ID": o.get("gatewayId"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
@@ -209,7 +219,7 @@ def create_connection_vnet(
         },
     }
 
-    response = client.post(f"/v1/connections", json=request_body)
+    response = client.post("/v1/connections", json=request_body)
 
     if response.status_code == 200:
         o = response.json()
@@ -218,13 +228,19 @@ def create_connection_vnet(
             "Connection Name": o.get("name"),
             "Gateway ID": o.get("gatewayId"),
             "Connectivity Type": o.get("connectivityType"),
-            "Connection Type": o.get("connectionDetails",{}).get("type"),
-            "Connection Path": o.get("connectionDetails",{}).get("path"),
+            "Connection Type": o.get("connectionDetails", {}).get("type"),
+            "Connection Path": o.get("connectionDetails", {}).get("path"),
             "Privacy Level": o.get("privacyLevel"),
-            "Credential Type": o.get("credentialDetails",{}).get("credentialType"),
-            "Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
-            "Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
-            "Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
+            "Credential Type": o.get("credentialDetails", {}).get("credentialType"),
+            "Single Sign On Type": o.get("credentialDetails", {}).get(
+                "singleSignOnType"
+            ),
+            "Connection Encryption": o.get("credentialDetails", {}).get(
+                "connectionEncryption"
+            ),
+            "Skip Test Connection": o.get("credentialDetails", {}).get(
+                "skipTestConnection"
+            ),
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)