From 06dc781db860918fbb4c5c356317390ef3e77ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20Zag=C3=B3rski?= <1507542+zbigg@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:50:48 +0200 Subject: [PATCH] Remote spatial index widgets #2 (#910) --- CHANGELOG.md | 2 + packages/react-api/src/types.d.ts | 1 + packages/react-redux/src/slices/cartoSlice.js | 3 + .../__tests__/models/utils.test.js | 25 +++++- .../react-widgets/src/hooks/useWidgetFetch.js | 34 ++------ packages/react-widgets/src/index.d.ts | 13 ++- packages/react-widgets/src/index.js | 1 + .../src/models/spatialFiltersResolution.js | 87 +++++++++++++++++++ packages/react-widgets/src/models/utils.d.ts | 24 +++-- packages/react-widgets/src/models/utils.js | 28 ++++-- 10 files changed, 175 insertions(+), 43 deletions(-) create mode 100644 packages/react-widgets/src/models/spatialFiltersResolution.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a58629a4..ca84356c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Not released +- Remote calculation for dynamic spatial index sources [#908](https://github.com/CartoDB/carto-react/pull/908) + ## 3.0.0 ### 3.0.0-alpha.21 (2024-09-26) diff --git a/packages/react-api/src/types.d.ts b/packages/react-api/src/types.d.ts index ccb84c4b4..1c56959aa 100644 --- a/packages/react-api/src/types.d.ts +++ b/packages/react-api/src/types.d.ts @@ -48,6 +48,7 @@ export type SourceProps = { spatialDataColumn?: string; spatialFiltersResolution?: number; aggregationExp?: string; + aggregationResLevel?: number; credentials?: Credentials; queryParameters?: QueryParameters; provider?: Provider; diff --git a/packages/react-redux/src/slices/cartoSlice.js b/packages/react-redux/src/slices/cartoSlice.js index 3efc33e90..a1bb50837 100644 --- a/packages/react-redux/src/slices/cartoSlice.js +++ b/packages/react-redux/src/slices/cartoSlice.js @@ -207,6 +207,7 @@ export const createCartoSlice = (initialState) => { * @param {number=} data.dataResolution - data resolution for spatial index data. * @param {number=} data.spatialFiltersResolution - spatial filters resolution for spatial index data. * @param {string=} data.aggregationExp - (optional) for spatial index data. + * @param {number=} data.aggregationResLevel - (optional) for spatial index data. * @param {string=} data.provider - (optional) type of the data warehouse. */ export const addSource = ({ @@ -224,6 +225,7 @@ export const addSource = ({ dataResolution, spatialFiltersResolution, aggregationExp, + aggregationResLevel, provider }) => ({ type: 'carto/addSource', @@ -238,6 +240,7 @@ export const addSource = ({ queryParameters, geoColumn, dataResolution, + aggregationResLevel, spatialDataType, spatialDataColumn, spatialFiltersResolution, diff --git a/packages/react-widgets/__tests__/models/utils.test.js b/packages/react-widgets/__tests__/models/utils.test.js index 208bf8a20..aadf9e4df 100644 --- a/packages/react-widgets/__tests__/models/utils.test.js +++ b/packages/react-widgets/__tests__/models/utils.test.js @@ -2,7 +2,8 @@ import { sourceAndFiltersToSQL, wrapModelCall, formatOperationColumn, - normalizeObjectKeys + normalizeObjectKeys, + isRemoteCalculationSupported } from '../../src/models/utils'; import { AggregationTypes, Provider, _filtersToSQL } from '@carto/react-core'; import { MAP_TYPES, API_VERSIONS } from '@carto/react-api'; @@ -41,6 +42,28 @@ const fromLocal = jest.fn(); const fromRemote = jest.fn(); describe('utils', () => { + describe('isRemoteCalculationSupported', () => { + test.each([ + ['v2', V2_SOURCE, false], + ['v3', V3_SOURCE, true], + ['v3', { ...V3_SOURCE, type: 'tileset' }, false], + ['v3/databricks', { ...V3_SOURCE, provider: 'databricks' }, false], + ['v3/databricksRest', { ...V3_SOURCE, provider: 'databricksRest' }, true], + ['v3/h3/no dataResolution', { ...V3_SOURCE, geoColumn: 'h3' }, false], + [ + 'v3/h3/with dataResolution', + { ...V3_SOURCE, geoColumn: 'h3', dataResolution: 5 }, + true + ], + [ + 'v3/quadbin/with dataResolution', + { ...V3_SOURCE, geoColumn: 'quadbin:abc', spatialFiltersResolution: 5 }, + true + ] + ])('works correctly for %s', (_, source, expected) => { + expect(isRemoteCalculationSupported({ source })).toEqual(expected); + }); + }); describe('wrapModelCall', () => { const cases = [ // source, global, remoteCalculation, expectedFn diff --git a/packages/react-widgets/src/hooks/useWidgetFetch.js b/packages/react-widgets/src/hooks/useWidgetFetch.js index fb31931e1..81762dc46 100644 --- a/packages/react-widgets/src/hooks/useWidgetFetch.js +++ b/packages/react-widgets/src/hooks/useWidgetFetch.js @@ -16,6 +16,7 @@ import { DEFAULT_INVALID_COLUMN_ERR } from '../widgets/utils/constants'; import useCustomCompareEffect from './useCustomCompareEffect'; import useWidgetSource from './useWidgetSource'; import { isRemoteCalculationSupported } from '../models/utils'; +import { getSpatialFiltersResolution } from '../models/spatialFiltersResolution'; export const WidgetStateType = { Loading: 'loading', @@ -57,19 +58,6 @@ export function selectGeometryToIntersect(global, viewport, spatialFilter) { } } -// stolen from deck.gl/modules/carto/src/layers/h3-tileset-2d.ts -const BIAS = 2; -export function getHexagonResolution(viewport, tileSize) { - // Difference in given tile size compared to deck's internal 512px tile size, - // expressed as an offset to the viewport zoom. - const zoomOffset = Math.log2(tileSize / 512); - const hexagonScaleFactor = (2 / 3) * (viewport.zoom - zoomOffset); - const latitudeScaleFactor = Math.log(1 / Math.cos((Math.PI * viewport.latitude) / 180)); - - // Clip and bias - return Math.max(0, Math.floor(hexagonScaleFactor + latitudeScaleFactor - BIAS)); -} - export default function useWidgetFetch( modelFn, { @@ -118,21 +106,15 @@ export default function useWidgetFetch( return source; } - if (source.spatialDataType === 'h3') { - const hexagonResolution = getHexagonResolution( - { zoom: viewState.zoom, latitude: viewState.latitude }, - source.dataResolution - ); - return { - ...source, - spatialFiltersResolution: Math.min(source.dataResolution, hexagonResolution) - }; - } - if (source.spatialDataType === 'quadbin') { - const quadsResolution = Math.floor(viewState.zoom); + if (source.spatialDataType === 'h3' || source.spatialDataType === 'quadbin') { + const spatialFiltersResolution = getSpatialFiltersResolution({ + source, + viewState, + spatialDataType: source.spatialDataType + }); return { ...source, - spatialFiltersResolution: Math.min(source.dataResolution, quadsResolution) + spatialFiltersResolution }; } return source; diff --git a/packages/react-widgets/src/index.d.ts b/packages/react-widgets/src/index.d.ts index b81187b32..4bc0c4dd3 100644 --- a/packages/react-widgets/src/index.d.ts +++ b/packages/react-widgets/src/index.d.ts @@ -25,7 +25,14 @@ export { default as BarWidget } from './widgets/BarWidget'; export { default as useSourceFilters } from './hooks/useSourceFilters'; export { default as FeatureSelectionWidget } from './widgets/FeatureSelectionWidget'; export { default as FeatureSelectionLayer } from './layers/FeatureSelectionLayer'; -export { default as useGeocoderWidgetController, setGeocoderResult } from './hooks/useGeocoderWidgetController'; +export { + default as useGeocoderWidgetController, + setGeocoderResult +} from './hooks/useGeocoderWidgetController'; export { WidgetState, WidgetStateType } from './types'; -export { isRemoteCalculationSupported as _isRemoteCalculationSupported, sourceAndFiltersToSQL as _sourceAndFiltersToSQL, getSqlEscapedSource as _getSqlEscapedSource } from './models/utils'; - +export { + isRemoteCalculationSupported as _isRemoteCalculationSupported, + sourceAndFiltersToSQL as _sourceAndFiltersToSQL, + getSqlEscapedSource as _getSqlEscapedSource, + getSpatialFiltersResolution as _getSpatialFiltersResolution +} from './models/utils'; diff --git a/packages/react-widgets/src/index.js b/packages/react-widgets/src/index.js index bf79ddc23..8e89f30d8 100644 --- a/packages/react-widgets/src/index.js +++ b/packages/react-widgets/src/index.js @@ -31,3 +31,4 @@ export { sourceAndFiltersToSQL as _sourceAndFiltersToSQL, getSqlEscapedSource as _getSqlEscapedSource } from './models/utils'; +export { getSpatialFiltersResolution as _getSpatialFiltersResolution } from './models/spatialFiltersResolution'; diff --git a/packages/react-widgets/src/models/spatialFiltersResolution.js b/packages/react-widgets/src/models/spatialFiltersResolution.js new file mode 100644 index 000000000..660591221 --- /dev/null +++ b/packages/react-widgets/src/models/spatialFiltersResolution.js @@ -0,0 +1,87 @@ +// stolen from deck.gl/modules/carto/src/layers/h3-tileset-2d.ts +const BIAS = 2; +function getHexagonResolution(viewport, tileSize) { + // Difference in given tile size compared to deck's internal 512px tile size, + // expressed as an offset to the viewport zoom. + const zoomOffset = Math.log2(tileSize / 512); + const hexagonScaleFactor = (2 / 3) * (viewport.zoom - zoomOffset); + const latitudeScaleFactor = Math.log(1 / Math.cos((Math.PI * viewport.latitude) / 180)); + + // Clip and bias + return Math.max(0, Math.floor(hexagonScaleFactor + latitudeScaleFactor - BIAS)); +} + +const maxH3SpatialFiltersResolutions = [ + [20, 14], + [19, 13], + [18, 12], + [17, 11], + [16, 10], + [15, 9], + [14, 8], + [13, 7], + [12, 7], + [11, 7], + [10, 6], + [9, 6], + [8, 5], + [7, 4], + [6, 4], + [5, 3], + [4, 2], + [3, 1], + [2, 1], + [1, 0] +]; + +const quadBinZoomMaxOffset = 4; + +const DEFAULT_TILE_SIZE = 512; +const DEFAULT_AGGREGATION_RES_LEVEL_H3 = 4; +const DEFAULT_AGGREGATION_RES_LEVEL_QUADBIN = 6; + +export function getSpatialFiltersResolution({ source, spatialDataType, viewState }) { + if (spatialDataType === 'geo') return undefined; + + const currentZoom = viewState.zoom ?? 1; + + const dataResolution = source.dataResolution ?? Number.MAX_VALUE; + + const aggregationResLevel = + source.aggregationResLevel ?? + (spatialDataType === 'h3' + ? DEFAULT_AGGREGATION_RES_LEVEL_H3 + : DEFAULT_AGGREGATION_RES_LEVEL_QUADBIN); + + const aggregationResLevelOffset = Math.max(0, Math.floor(aggregationResLevel)); + + const currentZoomInt = Math.ceil(currentZoom); + if (spatialDataType === 'h3') { + const tileSize = DEFAULT_TILE_SIZE; + const maxResolutionForZoom = + maxH3SpatialFiltersResolutions.find(([zoom]) => zoom === currentZoomInt)?.[1] ?? + Math.max(0, currentZoomInt - 3); + + const maxSpatialFiltersResolution = maxResolutionForZoom + ? Math.min(dataResolution, maxResolutionForZoom) + : dataResolution; + + const hexagonResolution = + getHexagonResolution( + { zoom: currentZoom, latitude: viewState.latitude }, + tileSize + ) + aggregationResLevelOffset; + + return Math.min(hexagonResolution, maxSpatialFiltersResolution); + } + + if (spatialDataType === 'quadbin') { + const maxResolutionForZoom = currentZoomInt + quadBinZoomMaxOffset; + const maxSpatialFiltersResolution = Math.min(dataResolution, maxResolutionForZoom); + + const quadsResolution = Math.floor(viewState.zoom) + aggregationResLevelOffset; + return Math.min(quadsResolution, maxSpatialFiltersResolution); + } + + return undefined; +} diff --git a/packages/react-widgets/src/models/utils.d.ts b/packages/react-widgets/src/models/utils.d.ts index de1d06827..ca3be1459 100644 --- a/packages/react-widgets/src/models/utils.d.ts +++ b/packages/react-widgets/src/models/utils.d.ts @@ -1,9 +1,21 @@ -import { SourceProps, MAP_TYPES } from "@carto/react-api"; -import { FiltersLogicalOperators, Provider, _FilterTypes, Provider } from "@carto/react-core"; -import { SourceFilters } from "@carto/react-redux"; +import { SourceProps, MAP_TYPES } from '@carto/react-api'; +import { FiltersLogicalOperators, Provider, _FilterTypes } from '@carto/react-core'; +import { SourceFilters, ViewState } from '@carto/react-redux'; -export function isRemoteCalculationSupported(prop: { source: SourceProps }): boolean +export function isRemoteCalculationSupported(prop: { source: SourceProps }): boolean; -export function sourceAndFiltersToSQL(props: { data: string, filters?: SourceFilters, filtersLogicalOperator?: FiltersLogicalOperators, provider: Provider, type: typeof MAP_TYPES }): string +export function sourceAndFiltersToSQL(props: { + data: string; + filters?: SourceFilters; + filtersLogicalOperator?: FiltersLogicalOperators; + provider: Provider; + type: typeof MAP_TYPES; +}): string; -export function getSqlEscapedSource(table: string, provider: Provider): string \ No newline at end of file +export function getSqlEscapedSource(table: string, provider: Provider): string; + +export function getSpatialFiltersResolution(props: { + source: SourceProps; + spatialDataType: string; + viewState: ViewState; +}): number; diff --git a/packages/react-widgets/src/models/utils.js b/packages/react-widgets/src/models/utils.js index 7a34358f3..67504ce68 100644 --- a/packages/react-widgets/src/models/utils.js +++ b/packages/react-widgets/src/models/utils.js @@ -7,16 +7,30 @@ import { import { FullyQualifiedName } from './fqn'; import { MAP_TYPES, API_VERSIONS } from '@carto/react-api'; +export { getSpatialFiltersResolution } from './spatialFiltersResolution'; + export function isRemoteCalculationSupported(props) { const { source } = props; - return ( - source && - source.type !== MAP_TYPES.TILESET && - source.credentials.apiVersion !== API_VERSIONS.V2 && - !(source.geoColumn && getSpatialIndexFromGeoColumn(source.geoColumn)) && - source.provider !== 'databricks' - ); + if ( + !source || + source.type === MAP_TYPES.TILESET || + source.credentials.apiVersion === API_VERSIONS.V2 || + source.provider === 'databricks' + ) { + return false; + } + + const isDynamicSpatialIndex = + source.geoColumn && getSpatialIndexFromGeoColumn(source.geoColumn); + if ( + isDynamicSpatialIndex && + !source.dataResolution && + !source.spatialFiltersResolution + ) { + return false; + } + return true; } export function wrapModelCall(props, fromLocal, fromRemote) {