diff --git a/docs/geospatialViewsWebPortal.md b/docs/geospatialViewsWebPortal.md index 02527293..7baa36dd 100644 --- a/docs/geospatialViewsWebPortal.md +++ b/docs/geospatialViewsWebPortal.md @@ -1,12 +1,20 @@ # Geospatial Views in KNIME Data Apps -In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the KNIME WebPortal -you need to add the following line to the [knime-server.config file](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file): + +The Geospatial Views load visualization libraries from the KNIME Business Hub or KNIME Server during viewing of the data. In addition, dependent on the view and settings they might need to download background tile images from the different providers such as MapBox or OpenstreetMap. By default, these downloads are prevented by the default security settings and need to be enabled as described in the following sections. + +## KNIME Business Hub +In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the [KNIME Business Hub](https://www.knime.com/knime-business-hub) +you need to add the following line to the [Content Security Policy for Data Apps](https://docs.knime.com/latest/business_hub_admin_guide/index.html#configure-browser-security): + ``` -com.knime.server.webportal.csp=default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' -data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org -stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; -font-src 'self' data:;worker-src blob: 'self'; +default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; font-src 'self' data:; ``` -This will allow the views to load the required libraries from the KNIME Server and the background tile images from the different providers such as MapBox or OpenstreetMap. +For further information see the [Configure Browser Security](https://docs.knime.com/latest/business_hub_admin_guide/index.html#configure-browser-security) section of the [KNIME Business Hub Admin Guide](https://docs.knime.com/latest/business_hub_admin_guide/index.html). +## KNIME Server +In order to use the Geospatial Views in a [KNIME Data App](https://www.knime.com/data-apps) on the KNIME Server +you need to add the following line to the [knime-server.config file](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file): +``` +com.knime.server.webportal.csp=default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self'; style-src 'unsafe-inline' 'self';img-src 'self' data: *.arcgisonline.com *.autonavi.com *.basemaps.cartocdn.com *.cloudfront.net/kepler.gl/ *.openrailwaymap.org *.openstreetmap.org stamen-tiles-a.a.ssl.fastly.net *.strava.com *.earthdata.nasa.gov; connect-src 'self' *.mapbox.com *.cloudfront.net/kepler.gl/; font-src 'self' data:;worker-src blob: 'self'; +``` For further information about the different options see the [WebPortal Admin Guide](https://docs.knime.com/latest/webportal_admin_guide/index.html#knime-server-configuration-file-options-webportal). diff --git a/knime_extension/geospatial_env.yml b/knime_extension/geospatial_env.yml index 43642e95..a90f2ac9 100644 --- a/knime_extension/geospatial_env.yml +++ b/knime_extension/geospatial_env.yml @@ -9,7 +9,7 @@ dependencies: - esda=2.4.3 - fiona=1.9.3 - folium=0.14.0 - - geopandas=0.13.0 + - geopandas=0.13.2 - geopy=2.3.0 - ipykernel=6.22.0 #required by kepler but not included in its dependencies it seems - jmespath=1.0.1 diff --git a/knime_extension/knime.yml b/knime_extension/knime.yml index 36f6e2a7..6eb51248 100644 --- a/knime_extension/knime.yml +++ b/knime_extension/knime.yml @@ -4,7 +4,7 @@ author: Lingbo Liu,Xiaokang Fu,Tobias Koetter vendor: SDL, Harvard, Cambridge US description: Geospatial Analytics Extension for KNIME # Human readable bundle name / description long_description: KNIME nodes for processing, analyzing and visualizing Geospatial data. -version: 1.1.0 # Version of this Python node extension +version: 1.1.1 # Version of this Python node extension license_file: LICENSE.TXT # Best practice: put your LICENSE.TXT next to the knime.yml; otherwise you would need to change to path/to/LICENSE.txt extension_module: src/geospatial_ext # The .py Python module containing the nodes of your extension env_yml_path: geospatial_env.yml # This is necessary for bundling, but not needed during development diff --git a/knime_extension/src/nodes/spatialmodels.py b/knime_extension/src/nodes/spatialmodels.py index 95c55013..f666d171 100644 --- a/knime_extension/src/nodes/spatialmodels.py +++ b/knime_extension/src/nodes/spatialmodels.py @@ -74,7 +74,11 @@ def get_id_col_parameter( ) class Spatial2SLSModel: """Spatial two stage least squares (S2SLS) with results and diagnostics. - Spatial two stage least squares (S2SLS) with results and diagnostics. More details can be found in the following reference, Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer. Dordrecht, 1988. + Spatial two stage least squares (S2SLS) with results and diagnostics. More details can be found in the following reference, Luc Anselin. + Spatial Econometrics: Methods and Models. Kluwer. Dordrecht, 1988. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ # input parameters @@ -243,7 +247,11 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2): ) class SpatialLagPanelModelwithFixedEffects: """Spatial Lag Panel Model with Fixed Effects. - Spatial Lag Panel Model with Fixed Effects. ML estimation of the fixed effects spatial lag model with all results and diagnostics. More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791. + Spatial Lag Panel Model with Fixed Effects. ML estimation of the fixed effects spatial lag model with all results and diagnostics. More details can be found at J. Paul Elhorst. + Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -383,7 +391,11 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2): ) class SpatialErrorPanelModelwithFixedEffects: """Spatial Error Panel Model with Fixed Effects node. - Spatial Error Panel Model with Fixed Effects node. ML estimation of the fixed effects spatial error model with all results and diagnostics. More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791. + Spatial Error Panel Model with Fixed Effects node. ML estimation of the fixed effects spatial error model with all results and diagnostics. + More details can be found at J. Paul Elhorst. Specification and estimation of spatial panel data models. International Regional Science Review, 26(3):244–268, 2003. doi:10.1177/0160017603253791. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -525,6 +537,9 @@ class GeographicallyWeightedRegression: """Geographically Weighted Regression node. Performs Geographically Weighted Regression (GWR), a local form of linear regression used to model spatially varying relationships. Can currently estimate Gaussian, Poisson, and logistic models(built on a GLM framework). More details can be found at [here](https://desktop.arcgis.com/en/arcmap/10.3/tools/spatial-statistics-toolbox/geographically-weighted-regression.htm). + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -665,6 +680,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1): class GeographicallyWeightedRegressionPredictor: """Geographically Weighted Regression Predictor node. Geographically Weighted Regression Predictor. It will predict the dependent variable using the model and the input table. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -733,7 +751,13 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, model): ) class MultiscaleGeographicallyWeightedRegression: """Multiscale Geographically Weighted Regression node. - Multiscale Geographically Weighted Regression estimation. More details can be found at A. Stewart Fotheringham, Wenbai Yang, and Wei Kang. Multiscale geographically weighted regression (mgwr). Annals of the American Association of Geographers, 107(6):1247–1265, 2017. URL: http://dx.doi.org/10.1080/24694452.2017.1352480, arXiv:http://dx.doi.org/10.1080/24694452.2017.1352480, doi:10.1080/24694452.2017.1352480. and Hanchen Yu, Alexander Stewart Fotheringham, Ziqi Li, Taylor Oshan, Wei Kang, and Levi John Wolf. Inference in multiscale geographically weighted regression. Geographical Analysis, 2019. URL: https://onlinelibrary.wiley.com/doi/abs/10.1111/gean.12189, arXiv:https://onlinelibrary.wiley.com/doi/pdf/10.1111/gean.12189, doi:10.1111/gean.12189. + Multiscale Geographically Weighted Regression estimation. + More details can be found at + 1. A. Stewart Fotheringham, Wenbai Yang, and Wei Kang. Multiscale geographically weighted regression (mgwr). Annals of the American Association of Geographers, 107(6):1247–1265, 2017. URL: http://dx.doi.org/10.1080/24694452.2017.1352480, arXiv:http://dx.doi.org/10.1080/24694452.2017.1352480, doi:10.1080/24694452.2017.1352480. and Hanchen Yu, Alexander Stewart Fotheringham, Ziqi Li, Taylor Oshan, Wei Kang, and Levi John Wolf. Inference in multiscale geographically weighted regression. Geographical Analysis, 2019. URL: https://onlinelibrary.wiley.com/doi/abs/10.1111/gean.12189, arXiv:https://onlinelibrary.wiley.com/doi/pdf/10.1111/gean.12189, doi:10.1111/gean.12189. + 2. https://pro.arcgis.com/en/pro-app/latest/tool-reference/spatial-statistics/how-multiscale-geographically-weighted-regression-mgwr-works.htm + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -752,7 +776,10 @@ class MultiscaleGeographicallyWeightedRegression: search_method = knext.StringParameter( "Search method", - "Bw search method: ‘golden’, ‘interval’", + """Bw search method: ‘golden’, ‘interval’. Golden Search— Determines either the number of neighbors or distance band for each + explanatory variable using the Golden Search algorithm. This method searches multiple combinations + of values for each explanatory variable between a specified minimum and maximum value. Intervals— Determines the number of neighbors or distance band for each + explanatory variable by incrementing the number of neighbors or distance band from a minimum value.""", default_value="golden", enum=["golden", "interval"], ) @@ -787,8 +814,8 @@ def execute(self, exec_context: knext.ExecutionContext, input_1): # Prepare Georgia dataset inputs g_y = gdf[self.dependent_variable].values.reshape((-1, 1)) g_X = gdf[self.independent_variables].values - u = gdf["geometry"].x - v = gdf["geometry"].y + u = gdf["geometry"].centroid.x + v = gdf["geometry"].centroid.y g_coords = list(zip(u, v)) # g_X = (g_X - g_X.mean(axis=0)) / g_X.std(axis=0) g_y = g_y.reshape((-1, 1)) @@ -958,6 +985,9 @@ class SpatialOLS: """Spatial OLS node. Ordinary least squares with results and diagnostics. More information can be found at [here](https://spreg.readthedocs.io/en/latest/generated/spreg.OLS.html) + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -1098,6 +1128,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2): class SpatialML_Lag: """Spatial ML_Lag. ML estimation of the spatial lag model with all results and diagnostics. More details can be found at Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer, Dordrecht, 1988. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() @@ -1237,6 +1270,9 @@ def execute(self, exec_context: knext.ExecutionContext, input_1, input_2): class SpatialML_Error: """Spatial ML_Error. ML estimation of the spatial error model with all results and diagnostics. More details can be found at Luc Anselin. Spatial Econometrics: Methods and Models. Kluwer, Dordrecht, 1988. + + **Note:** The input table should not contain missing values. You can use the + [Missing Value](https://hub.knime.com/knime/extensions/org.knime.features.base/latest/org.knime.base.node.preproc.pmml.missingval.compute.MissingValueHandlerNodeFactory/) node to replace them. """ geo_col = knut.geo_col_parameter() diff --git a/knime_extension/src/nodes/spatialnetwork.py b/knime_extension/src/nodes/spatialnetwork.py index 3019640f..7e59558e 100644 --- a/knime_extension/src/nodes/spatialnetwork.py +++ b/knime_extension/src/nodes/spatialnetwork.py @@ -859,7 +859,7 @@ def split_line(self, line, pps): line = snap(line, pps, 1e-8) # slow? try: - new_lines = list(split(line, pps)) # split into segments + new_lines = list(split(line, pps).geoms) # split into segments return new_lines except TypeError as e: print("Error when splitting line: {}\n{}\n{}\n".format(e, line, pps)) @@ -949,7 +949,7 @@ def ckd_nearest(self, gdA, gdB): return gdf # check isolated node - def connect_graph(self, G, threshold=1): + def connect_graph(self, G, threshold=0.1): import math import networkx as nx from shapely.geometry import LineString @@ -1048,8 +1048,9 @@ def execute(self, exec_context: knext.ExecutionContext, input1, input2, input3): o_gdf = knut.load_geo_data_frame(input1, self.o_geo_col, exec_context) d_gdf = knut.load_geo_data_frame(input2, self.d_geo_col, exec_context) r_gdf = knut.load_geo_data_frame(input3, self.r_geo_col, exec_context) - - r_gdf = r_gdf.to_crs(3857) + if not r_gdf.crs.is_projected: + r_gdf = r_gdf.to_crs(3857) + knut.LOGGER.warning("Road not projected. Using EPSG 3857 as default.") o_gdf = o_gdf.to_crs(r_gdf.crs) d_gdf = d_gdf.to_crs(r_gdf.crs) diff --git a/knime_extension/src/nodes/spatialstatistics.py b/knime_extension/src/nodes/spatialstatistics.py index eb401796..cfd18392 100644 --- a/knime_extension/src/nodes/spatialstatistics.py +++ b/knime_extension/src/nodes/spatialstatistics.py @@ -118,7 +118,6 @@ class spatialWeights: "Lattice", "K nearest", "Lattice", - "K nearest", "Kernel", "Get spatial weights matrix from file", ], @@ -231,6 +230,12 @@ def execute(self, exec_context: knext.ExecutionContext, input_1): wname = "Inverse Distance" w.transform = "r" if self.category == "Binary Distance Band": + + import util.projection as kproj + + crs = gdf.crs + if (crs is not None) and (kproj.is_geographic(crs)): + gdf = gdf.to_crs("EPSG:3857") w = libpysal.weights.DistanceBand.from_dataframe( gdf, self.Threshold, binary=True ) @@ -276,7 +281,7 @@ def execute(self, exec_context: knext.ExecutionContext, input_1): # path = flow_variables["knime.workspace"] + os.sep +"spatialweights.gal" # w.to_file(path) # flow_variables["weights"] =path - out = w.to_adjlist() + out = w.to_adjlist(drop_islands=False) if "none" not in str(self.id_col).lower(): # get index id map diff --git a/knime_extension/src/nodes/spatialtool.py b/knime_extension/src/nodes/spatialtool.py index 4880271d..8a69adc7 100644 --- a/knime_extension/src/nodes/spatialtool.py +++ b/knime_extension/src/nodes/spatialtool.py @@ -843,7 +843,7 @@ def configure(self, configure_context, o_schema, d_schema): knut.column_exists(self.o_id_col, o_schema) o_id_type = o_schema[self.o_id_col].ktype knut.column_exists(self.d_id_col, d_schema) - d_id_type = o_schema[self.d_id_col].ktype + d_id_type = d_schema[self.d_id_col].ktype return knext.Schema.from_columns( [ knext.Column(o_id_type, self.__COL_ORIGIN), @@ -863,9 +863,11 @@ def execute(self, exec_context: knext.ExecutionContext, o_input, d_input): ) # rename the id columns to origin and destination and the geometry to geometry o_gdf.rename(columns={self.o_id_col: self.__COL_ORIGIN}, inplace=True) - o_gdf.rename_geometry("geometry", inplace=True) + if o_gdf.geometry.name != "geometry": + o_gdf.rename_geometry("geometry", inplace=True) d_gdf.rename(columns={self.d_id_col: self.__COL_DESTINATION}, inplace=True) - d_gdf.rename_geometry("geometry", inplace=True) + if d_gdf.geometry.name != "geometry": + d_gdf.rename_geometry("geometry", inplace=True) helper = kproj.Distance(self.unit, True) helper.pre_processing(exec_context, o_gdf, True) diff --git a/knime_extension/src/nodes/transform.py b/knime_extension/src/nodes/transform.py index eb9f9b30..9543e61c 100644 --- a/knime_extension/src/nodes/transform.py +++ b/knime_extension/src/nodes/transform.py @@ -350,7 +350,7 @@ class PointsToLineNode: "Group column", "Select the group column (string) as group id for points.", # Allow only string columns - column_filter=knut.is_string, + column_filter=knut.is_int_or_string, include_row_key=False, include_none_column=False, ) @@ -369,7 +369,7 @@ def configure(self, configure_context, input_schema): configure_context, self.geo_col, input_schema, knut.is_geo_point ) self.group_col = knut.column_exists_or_preset( - configure_context, self.group_col, input_schema, knut.is_string + configure_context, self.group_col, input_schema, knut.is_int_or_string ) self.seiral_col = knut.column_exists_or_preset( configure_context, self.seiral_col, input_schema, knut.is_numeric diff --git a/knime_extension/src/nodes/visualize.py b/knime_extension/src/nodes/visualize.py index fc3d3690..82f22cf8 100644 --- a/knime_extension/src/nodes/visualize.py +++ b/knime_extension/src/nodes/visualize.py @@ -342,7 +342,6 @@ class BaseMapSettings: "Stamen TopOSMFeatures", "Stamen TopOSMRelief", "Stamen Watercolor", - "Stamen Watercolor", "Strava All", "Strava Ride", "Strava Run", diff --git a/knime_extension/src/util/knime_utils.py b/knime_extension/src/util/knime_utils.py index eb069217..9d9cefaa 100644 --- a/knime_extension/src/util/knime_utils.py +++ b/knime_extension/src/util/knime_utils.py @@ -150,7 +150,10 @@ def is_int(column: knext.Column) -> bool: Checks if column is integer. @return: True if Column is integer """ - return column.ktype == knext.int32() + return column.ktype in [ + knext.int32(), + knext.int64(), + ] def is_string(column: knext.Column) -> bool: @@ -177,6 +180,18 @@ def is_numeric_or_string(column: knext.Column) -> bool: return boolean_or(is_numeric, is_string)(column) +def is_int_or_string(column: knext.Column) -> bool: + """ + Checks if column is int or string + @return: True if Column is numeric or string + """ + return column.ktype in [ + knext.int32(), + knext.int64(), + knext.string(), + ] + + def is_binary(column: knext.Column) -> bool: """ Checks if column is binary