Add temporary workaround for SPARK-48710

https://issues.apache.org/jira/browse/SPARK-48710

hamilton/registry.py

@@ -3,6 +3,9 @@
 import importlib
 import logging
 from typing import Any, Dict, Optional, Type
+from unittest.mock import patch
+
+import numpy as np
 
 logger = logging.getLogger(__name__)
 
@@ -78,7 +81,9 @@ def load_extension(plugin_module: str):
 
     :param plugin_module: the module name sans .py. e.g. pandas, polars, pyspark_pandas.
     """
-    mod = importlib.import_module(f"hamilton.plugins.{plugin_module}_extensions")
+    with patch.object(np, "NaN", np.nan):
+        # Workaround for https://issues.apache.org/jira/browse/SPARK-48710
+        mod = importlib.import_module(f"hamilton.plugins.{plugin_module}_extensions")
     # We have various plugin extensions. We default to assuming it's a dataframe extension with columns,
     # unless it explicitly says it's not.
     # We need to check the following if we are to enable `@extract_columns` for example.