Merge pull request #246 from spatial-data-lab/main

Prepare 1.1.1 release

docs/geospatialViewsWebPortal.md

@@ -1,12 +1,20 @@
 # Geospatial Views in KNIME Data Apps
-In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the KNIME WebPortal 
-you need to add the following line to the [knime-server.config file](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file):
+
+The Geospatial Views load visualization libraries from the KNIME Business Hub or KNIME Server during viewing of the data. In addition, dependent on the view and settings they might need to download background tile images from the different providers such as MapBox or OpenstreetMap. By default, these downloads are prevented by the default security settings and need to be enabled as described in the following sections.
+
+## KNIME Business Hub
+In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the [KNIME Business Hub](https://www.knime.com/knime-business-hub)
+you need to add the following line to the [Content Security Policy for Data Apps](https://docs.knime.com/latest/business_hub_admin_guide/index.html#configure-browser-security):
+
 ```
-com.knime.server.webportal.csp=default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' 
-data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org 
-stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; 
-font-src 'self' data:;worker-src blob: 'self';
+default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; font-src 'self' data:;
 ```
-This will allow the views to load the required libraries from the KNIME Server and the background tile images from the different providers such as MapBox or OpenstreetMap.
+For further information see the [Configure Browser Security](https://docs.knime.com/latest/business_hub_admin_guide/index.html#configure-browser-security) section of the [KNIME Business Hub Admin Guide](https://docs.knime.com/latest/business_hub_admin_guide/index.html).
 
+## KNIME Server
+In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the KNIME Server 
+you need to add the following line to the [knime-server.config file](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file):
+```
+com.knime.server.webportal.csp=default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; font-src 'self' data:;worker-src blob: 'self';
+```
 For further information about the different options see the [WebPortal Admin Guide](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file-options-webportal).

knime_extension/geospatial_env.yml

@@ -9,7 +9,7 @@ dependencies:
   - esda=2.4.3
   - fiona=1.9.3
   - folium=0.14.0
-  - geopandas=0.13.0
+  - geopandas=0.13.2
   - geopy=2.3.0
   - ipykernel=6.22.0 #required by kepler but not included in its dependencies it seems
   - jmespath=1.0.1

knime_extension/knime.yml

@@ -4,7 +4,7 @@ author: Lingbo Liu,Xiaokang Fu,Tobias Koetter
 vendor: SDL, Harvard, Cambridge US
 description: Geospatial Analytics Extension for KNIME # Human readable bundle name / description
 long_description: KNIME nodes for processing, analyzing and visualizing Geospatial data.
-version: 1.1.0 # Version of this Python node extension
+version: 1.1.1 # Version of this Python node extension
 license_file: LICENSE.TXT # Best practice: put your LICENSE.TXT next to the knime.yml; otherwise you would need to change to path/to/LICENSE.txt
 extension_module: src/geospatial_ext # The .py Python module containing the nodes of your extension
 env_yml_path: geospatial_env.yml # This is necessary for bundling, but not needed during development

knime_extension/src/nodes/spatialmodels.py

@@ -74,7 +74,11 @@ def get_id_col_parameter(
 )
 class Spatial2SLSModel:
     """Spatial two stage least squares (S2SLS) with results and diagnostics.
-    Spatial two stage least squares (S2SLS) with results and diagnostics. More details can be found in the following reference, Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer. Dordrecht, 1988.
+    Spatial two stage least squares (S2SLS) with results and diagnostics. More details can be found in the following reference, Luc Anselin.
+    Spatial Econometrics: Methods and Models. Kluwer. Dordrecht, 1988.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     # input parameters
@@ -243,7 +247,11 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2):
 )
 class SpatialLagPanelModelwithFixedEffects:
     """Spatial Lag Panel Model with Fixed Effects.
-    Spatial Lag Panel Model with Fixed Effects. ML estimation of the fixed effects spatial lag model with all results and diagnostics. More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791.
+    Spatial Lag Panel Model with Fixed Effects. ML estimation of the fixed effects spatial lag model with all results and diagnostics. More details can be found at J. Paul Elhorst.
+    Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -383,7 +391,11 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2):
 )
 class SpatialErrorPanelModelwithFixedEffects:
     """Spatial Error Panel Model with Fixed Effects node.
-    Spatial Error Panel Model with Fixed Effects node. ML estimation of the fixed effects spatial error model with all results and diagnostics. More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791.
+    Spatial Error Panel Model with Fixed Effects node. ML estimation of the fixed effects spatial error model with all results and diagnostics.
+    More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -525,6 +537,9 @@ class GeographicallyWeightedRegression:
     """Geographically Weighted Regression node.
     Performs Geographically Weighted Regression (GWR), a local form of linear regression used to model spatially varying relationships. Can currently estimate Gaussian, Poisson, and logistic models(built on a GLM framework).
     More details can be found at [here](https://desktop.arcgis.com/en/arcmap/10.3/tools/spatial-statistics-toolbox/geographically-weighted-regression.htm).
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -665,6 +680,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1):
 class GeographicallyWeightedRegressionPredictor:
     """Geographically Weighted Regression Predictor node.
     Geographically Weighted Regression Predictor. It will predict the dependent variable using the model and the input table.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -733,7 +751,13 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, model):
 )
 class MultiscaleGeographicallyWeightedRegression:
     """Multiscale Geographically Weighted Regression node.
-    Multiscale Geographically Weighted Regression estimation. More details can be found at A. Stewart Fotheringham, Wenbai Yang, and Wei Kang. Multiscale geographically weighted regression (mgwr). Annals of the American Association of Geographers, 107(6):1247–1265, 2017. URL: http://dx.doi.org/10.1080/24694452.2017.1352480, arXiv:http://dx.doi.org/10.1080/24694452.2017.1352480, doi:10.1080/24694452.2017.1352480. and Hanchen Yu, Alexander Stewart Fotheringham, Ziqi Li, Taylor Oshan, Wei Kang, and Levi John Wolf. Inference in multiscale geographically weighted regression. Geographical Analysis, 2019. URL: https://onlinelibrary.wiley.com/doi/abs/10.1111/gean.12189, arXiv:https://onlinelibrary.wiley.com/doi/pdf/10.1111/gean.12189, doi:10.1111/gean.12189.
+    Multiscale Geographically Weighted Regression estimation.
+    More details can be found at
+    1. A. Stewart Fotheringham, Wenbai Yang, and Wei Kang. Multiscale geographically weighted regression (mgwr). Annals of the American Association of Geographers, 107(6):1247–1265, 2017. URL: http://dx.doi.org/10.1080/24694452.2017.1352480, arXiv:http://dx.doi.org/10.1080/24694452.2017.1352480, doi:10.1080/24694452.2017.1352480. and Hanchen Yu, Alexander Stewart Fotheringham, Ziqi Li, Taylor Oshan, Wei Kang, and Levi John Wolf. Inference in multiscale geographically weighted regression. Geographical Analysis, 2019. URL: https://onlinelibrary.wiley.com/doi/abs/10.1111/gean.12189, arXiv:https://onlinelibrary.wiley.com/doi/pdf/10.1111/gean.12189, doi:10.1111/gean.12189.
+    2. https://pro.arcgis.com/en/pro-app/latest/tool-reference/spatial-statistics/how-multiscale-geographically-weighted-regression-mgwr-works.htm
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -752,7 +776,10 @@ class MultiscaleGeographicallyWeightedRegression:
 
     search_method = knext.StringParameter(
         "Search method",
-        "Bw search method: ‘golden’, ‘interval’",
+        """Bw search method: ‘golden’, ‘interval’. Golden Search— Determines either the number of neighbors or distance band for each 
+        explanatory variable using the Golden Search algorithm. This method searches multiple combinations 
+        of values for each explanatory variable between a specified minimum and maximum value. Intervals— Determines the number of neighbors or distance band for each 
+        explanatory variable by incrementing the number of neighbors or distance band from a minimum value.""",
         default_value="golden",
         enum=["golden", "interval"],
     )
@@ -787,8 +814,8 @@ def execute(self, exec_context: knext.ExecutionContext, input_1):
         # Prepare Georgia dataset inputs
         g_y = gdf[self.dependent_variable].values.reshape((-1, 1))
         g_X = gdf[self.independent_variables].values
-        u = gdf["geometry"].x
-        v = gdf["geometry"].y
+        u = gdf["geometry"].centroid.x
+        v = gdf["geometry"].centroid.y
         g_coords = list(zip(u, v))
         # g_X = (g_X - g_X.mean(axis=0)) / g_X.std(axis=0)
         g_y = g_y.reshape((-1, 1))
@@ -958,6 +985,9 @@ class SpatialOLS:
     """Spatial OLS node.
     Ordinary least squares with results and diagnostics. More information can be found at
     [here](https://spreg.readthedocs.io/en/latest/generated/spreg.OLS.html)
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -1098,6 +1128,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2):
 class SpatialML_Lag:
     """Spatial ML_Lag.
     ML estimation of the spatial lag model with all results and diagnostics. More details can be found at Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer, Dordrecht, 1988.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()
@@ -1237,6 +1270,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2):
 class SpatialML_Error:
     """Spatial ML_Error.
     ML estimation of the spatial error model with all results and diagnostics. More details can be found at Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer, Dordrecht, 1988.
+
+    **Note:** The input table should not contain missing values. You can use the
+    [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them.
     """
 
     geo_col = knut.geo_col_parameter()

knime_extension/src/nodes/spatialnetwork.py

@@ -859,7 +859,7 @@ def split_line(self, line, pps):
 
         line = snap(line, pps, 1e-8)  # slow?
         try:
-            new_lines = list(split(line, pps))  # split into segments
+            new_lines = list(split(line, pps).geoms)  # split into segments
             return new_lines
         except TypeError as e:
             print("Error when splitting line: {}\n{}\n{}\n".format(e, line, pps))
@@ -949,7 +949,7 @@ def ckd_nearest(self, gdA, gdB):
         return gdf
 
     # check isolated node
-    def connect_graph(self, G, threshold=1):
+    def connect_graph(self, G, threshold=0.1):
         import math
         import networkx as nx
         from shapely.geometry import LineString
@@ -1048,8 +1048,9 @@ def execute(self, exec_context: knext.ExecutionContext, input1, input2, input3):
         o_gdf = knut.load_geo_data_frame(input1, self.o_geo_col, exec_context)
         d_gdf = knut.load_geo_data_frame(input2, self.d_geo_col, exec_context)
         r_gdf = knut.load_geo_data_frame(input3, self.r_geo_col, exec_context)
-
-        r_gdf = r_gdf.to_crs(3857)
+        if not r_gdf.crs.is_projected:
+            r_gdf = r_gdf.to_crs(3857)
+            knut.LOGGER.warning("Road not projected. Using EPSG 3857 as default.")
         o_gdf = o_gdf.to_crs(r_gdf.crs)
         d_gdf = d_gdf.to_crs(r_gdf.crs)
 

knime_extension/src/nodes/spatialstatistics.py

@@ -118,7 +118,6 @@ class spatialWeights:
             "Lattice",
             "K nearest",
             "Lattice",
-            "K nearest",
             "Kernel",
             "Get spatial weights matrix from file",
         ],
@@ -231,6 +230,12 @@ def execute(self, exec_context: knext.ExecutionContext, input_1):
             wname = "Inverse Distance"
             w.transform = "r"
         if self.category == "Binary Distance Band":
+
+            import util.projection as kproj 
+
+            crs = gdf.crs
+            if (crs is not None) and (kproj.is_geographic(crs)):
+                gdf = gdf.to_crs("EPSG:3857")
             w = libpysal.weights.DistanceBand.from_dataframe(
                 gdf, self.Threshold, binary=True
             )
@@ -276,7 +281,7 @@ def execute(self, exec_context: knext.ExecutionContext, input_1):
         # path = flow_variables["knime.workspace"] + os.sep +"spatialweights.gal"
         # w.to_file(path)
         # flow_variables["weights"] =path
-        out = w.to_adjlist()
+        out = w.to_adjlist(drop_islands=False)
 
         if "none" not in str(self.id_col).lower():
             # get index id map

knime_extension/src/nodes/spatialtool.py

@@ -843,7 +843,7 @@ def configure(self, configure_context, o_schema, d_schema):
         knut.column_exists(self.o_id_col, o_schema)
         o_id_type = o_schema[self.o_id_col].ktype
         knut.column_exists(self.d_id_col, d_schema)
-        d_id_type = o_schema[self.d_id_col].ktype
+        d_id_type = d_schema[self.d_id_col].ktype
         return knext.Schema.from_columns(
             [
                 knext.Column(o_id_type, self.__COL_ORIGIN),
@@ -863,9 +863,11 @@ def execute(self, exec_context: knext.ExecutionContext, o_input, d_input):
         )
         # rename the id columns to origin and destination and the geometry to geometry
         o_gdf.rename(columns={self.o_id_col: self.__COL_ORIGIN}, inplace=True)
-        o_gdf.rename_geometry("geometry", inplace=True)
+        if o_gdf.geometry.name != "geometry":
+            o_gdf.rename_geometry("geometry", inplace=True)
         d_gdf.rename(columns={self.d_id_col: self.__COL_DESTINATION}, inplace=True)
-        d_gdf.rename_geometry("geometry", inplace=True)
+        if d_gdf.geometry.name != "geometry":
+            d_gdf.rename_geometry("geometry", inplace=True)
 
         helper = kproj.Distance(self.unit, True)
         helper.pre_processing(exec_context, o_gdf, True)

knime_extension/src/nodes/transform.py

@@ -350,7 +350,7 @@ class PointsToLineNode:
         "Group column",
         "Select the group column (string) as group id for points.",
         # Allow only string columns
-        column_filter=knut.is_string,
+        column_filter=knut.is_int_or_string,
         include_row_key=False,
         include_none_column=False,
     )
@@ -369,7 +369,7 @@ def configure(self, configure_context, input_schema):
             configure_context, self.geo_col, input_schema, knut.is_geo_point
         )
         self.group_col = knut.column_exists_or_preset(
-            configure_context, self.group_col, input_schema, knut.is_string
+            configure_context, self.group_col, input_schema, knut.is_int_or_string
         )
         self.seiral_col = knut.column_exists_or_preset(
             configure_context, self.seiral_col, input_schema, knut.is_numeric

knime_extension/src/nodes/visualize.py

@@ -342,7 +342,6 @@ class BaseMapSettings:
             "Stamen TopOSMFeatures",
             "Stamen TopOSMRelief",
             "Stamen Watercolor",
-            "Stamen Watercolor",
             "Strava All",
             "Strava Ride",
             "Strava Run",

knime_extension/src/util/knime_utils.py

@@ -150,7 +150,10 @@ def is_int(column: knext.Column) -> bool:
     Checks if column is integer.
     @return: True if Column is integer
     """
-    return column.ktype == knext.int32()
+    return column.ktype in [
+        knext.int32(),
+        knext.int64(),
+    ]
 
 
 def is_string(column: knext.Column) -> bool:
@@ -177,6 +180,18 @@ def is_numeric_or_string(column: knext.Column) -> bool:
     return boolean_or(is_numeric, is_string)(column)
 
 
+def is_int_or_string(column: knext.Column) -> bool:
+    """
+    Checks if column is int or string
+    @return: True if Column is numeric or string
+    """
+    return column.ktype in [
+        knext.int32(),
+        knext.int64(),
+        knext.string(),
+    ]
+
+
 def is_binary(column: knext.Column) -> bool:
     """
     Checks if column is binary