From 206ad0c923a43919f0c81dacfb01239e636e42a1 Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 22:01:26 +0100 Subject: [PATCH 1/7] Use Heatmap for large location histories Signed-off-by: Arne Hamann --- src/components/map/DeviceLayer.vue | 6 +-- src/components/map/DevicesLayer.vue | 60 +++++++++++++++++++-- src/components/map/LHeatMap.vue | 84 +++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 src/components/map/LHeatMap.vue diff --git a/src/components/map/DeviceLayer.vue b/src/components/map/DeviceLayer.vue index b4bfc9bdb..604caaf48 100644 --- a/src/components/map/DeviceLayer.vue +++ b/src/components/map/DeviceLayer.vue @@ -43,12 +43,12 @@ @mouseover="deviceLastPointMouseover" /> - - + @@ -25,11 +29,14 @@ import DeviceHoverMarker from './DeviceHoverMarker' import optionsController from '../../optionsController' import moment from '@nextcloud/moment' +import { binSearch } from '../../utils/common' +import LHeatMap from './LHeatMap' export default { name: 'DevicesLayer', components: { LFeatureGroup, + LHeatMap, DeviceLayer, DeviceHoverMarker, }, @@ -57,6 +64,13 @@ export default { data() { return { + optionsHeatMap: { + // minOpacity: null, + // maxZoom: null, + radius: 15, + blur: 10, + gradient: { 0.4: 'blue', 0.65: 'lime', 1: 'red' }, + }, optionValues: optionsController.optionValues, hoverPoint: null, } @@ -66,14 +80,50 @@ export default { displayedDevices() { return this.devices.filter(d => d.enabled) }, + enabledDevices() { + return this.devices.map(d => d.enabled) + }, + displayedDevicesHistories() { + return this.devices.map(d => d.enabled && d.historyEnabled) + }, + points() { + return this.devices.reduce((points, device) => { + if (device.enabled && device.historyEnabled) { + const lastNullIndex = binSearch(device.points, (p) => !p.timestamp) + const firstShownIndex = binSearch(device.points, (p) => (p.timestamp || 0) < this.start) + 1 + const lastShownIndex = binSearch(device.points, (p) => (p.timestamp || 0) < this.end) + const filteredDevicePoints = [ + ...device.points.slice(0, lastNullIndex + 1), + ...device.points.slice(firstShownIndex, lastShownIndex + 1), + ] + if (filteredDevicePoints >= 2500) { + const deviceLatLngs = filteredDevicePoints.map((p) => [p.lat, p.lng]) + points.push(...deviceLatLngs) + } + } + return points + }, []) + }, }, watch: { - devices: { - handler() { - this.hoverPoint = null - }, - deep: true, + enabledDevices() { + this.hoverPoint = null + }, + displayedDevicesHistories() { + if (this.$refs.devicesHeatMap) { + this.$refs.devicesHeatMap.setLatLngs(this.points) + } + }, + start() { + if (this.$refs.devicesHeatMap) { + this.$refs.devicesHeatMap.setLatLngs(this.points) + } + }, + end() { + if (this.$refs.devicesHeatMap) { + this.$refs.devicesHeatMap.setLatLngs(this.points) + } }, }, diff --git a/src/components/map/LHeatMap.vue b/src/components/map/LHeatMap.vue new file mode 100644 index 000000000..1f9a1e7a3 --- /dev/null +++ b/src/components/map/LHeatMap.vue @@ -0,0 +1,84 @@ + + + + From dc6dcec70612e33946e96fdc7020886c9e61d07f Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 22:02:54 +0100 Subject: [PATCH 2/7] Include devices in timeslider range calculation, don't enable histories on pageload Signed-off-by: Arne Hamann --- src/views/App.vue | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/views/App.vue b/src/views/App.vue index 8f1e93c6e..bab4b0255 100644 --- a/src/views/App.vue +++ b/src/views/App.vue @@ -342,6 +342,7 @@ export default { this.minPhotoTimestamp, this.minFavoriteTimestamp, this.minTrackTimestamp, + this.minDevicesTimestamp, ) || 0 }, maxDataTimestamp() { @@ -409,6 +410,15 @@ export default { ? this.trackDates[0] + 100 : moment().unix() + 100 }, + minDevicesTimestamp() { + return this.devices.reduce((min, device) => { + if (device.enabled && device.historyEnabled) { + const lastNullIndex = binSearch(device.points, (p) => !p.timestamp) + min = Math.min(min, device.points[lastNullIndex + 1].timestamp) + } + return min + }, moment().unix() + 100) + }, // displayed data displayedTracks() { return this.sliderEnabled @@ -1965,7 +1975,7 @@ export default { ...device, loading: false, enabled: false, - historyEnabled: optionsController.enabledDeviceLines.includes(device.id), + historyEnabled: false // optionsController.enabledDeviceLines.includes(device.id), } }) this.devices.forEach((device) => { @@ -2002,12 +2012,13 @@ export default { }, disableDevice(device) { device.enabled = false + device.historyEnabled = false this.saveEnabledDevices() }, getDevice(device, enable = false, save = true, zoom = false) { device.loading = true network.getDevice(device.id, this.myMapId).then((response) => { - this.$set(device, 'points', response.data.sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0))) + this.$set(device, 'points', response.data /*.sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0))*/) if (enable) { device.enabled = true } From 5c680f7f9ef5752fe9a49f2b8f21ca48b983ce8e Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 22:03:30 +0100 Subject: [PATCH 3/7] install Leaflet.heat Signed-off-by: Arne Hamann --- package-lock.json | 11 +++++++++++ package.json | 1 + 2 files changed, 12 insertions(+) diff --git a/package-lock.json b/package-lock.json index bb66e5c7b..e9ba56afa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "leaflet-mouse-position": "^1.0.4", "leaflet-routing-machine": "^3.2.12", "leaflet.featuregroup.subgroup": "^1.0.2", + "leaflet.heat": "^0.2.0", "leaflet.locatecontrol": "^0.79.0", "leaflet.markercluster": "^1.5.3", "lrm-graphhopper": "^1.3.0", @@ -8359,6 +8360,11 @@ "resolved": "https://registry.npmjs.org/leaflet.featuregroup.subgroup/-/leaflet.featuregroup.subgroup-1.0.2.tgz", "integrity": "sha1-/7yW6m20GJO21ftgugnxOHbEXSg=" }, + "node_modules/leaflet.heat": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/leaflet.heat/-/leaflet.heat-0.2.0.tgz", + "integrity": "sha512-Cd5PbAA/rX3X3XKxfDoUGi9qp78FyhWYurFg3nsfhntcM/MCNK08pRkf4iEenO1KNqwVPKCmkyktjW3UD+h9bQ==" + }, "node_modules/leaflet.locatecontrol": { "version": "0.79.0", "resolved": "https://registry.npmjs.org/leaflet.locatecontrol/-/leaflet.locatecontrol-0.79.0.tgz", @@ -19390,6 +19396,11 @@ "resolved": "https://registry.npmjs.org/leaflet.featuregroup.subgroup/-/leaflet.featuregroup.subgroup-1.0.2.tgz", "integrity": "sha1-/7yW6m20GJO21ftgugnxOHbEXSg=" }, + "leaflet.heat": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/leaflet.heat/-/leaflet.heat-0.2.0.tgz", + "integrity": "sha512-Cd5PbAA/rX3X3XKxfDoUGi9qp78FyhWYurFg3nsfhntcM/MCNK08pRkf4iEenO1KNqwVPKCmkyktjW3UD+h9bQ==" + }, "leaflet.locatecontrol": { "version": "0.79.0", "resolved": "https://registry.npmjs.org/leaflet.locatecontrol/-/leaflet.locatecontrol-0.79.0.tgz", diff --git a/package.json b/package.json index 3c90b96cf..379055f79 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "leaflet-mouse-position": "^1.0.4", "leaflet-routing-machine": "^3.2.12", "leaflet.featuregroup.subgroup": "^1.0.2", + "leaflet.heat": "^0.2.0", "leaflet.locatecontrol": "^0.79.0", "leaflet.markercluster": "^1.5.3", "lrm-graphhopper": "^1.3.0", From 0a72c931460d187ab3fe4715f9e0a9019105f70a Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 22:41:25 +0100 Subject: [PATCH 4/7] Show only device with more then 2500 points on heatmap (others get a track) Signed-off-by: Arne Hamann --- src/components/map/DevicesLayer.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/map/DevicesLayer.vue b/src/components/map/DevicesLayer.vue index 67dc9a085..b481280cf 100644 --- a/src/components/map/DevicesLayer.vue +++ b/src/components/map/DevicesLayer.vue @@ -92,11 +92,11 @@ export default { const lastNullIndex = binSearch(device.points, (p) => !p.timestamp) const firstShownIndex = binSearch(device.points, (p) => (p.timestamp || 0) < this.start) + 1 const lastShownIndex = binSearch(device.points, (p) => (p.timestamp || 0) < this.end) - const filteredDevicePoints = [ - ...device.points.slice(0, lastNullIndex + 1), - ...device.points.slice(firstShownIndex, lastShownIndex + 1), - ] - if (filteredDevicePoints >= 2500) { + if (lastNullIndex + 1 + lastShownIndex - firstShownIndex + 1 > 2500) { + const filteredDevicePoints = [ + ...device.points.slice(0, lastNullIndex + 1), + ...device.points.slice(firstShownIndex, lastShownIndex + 1), + ] const deviceLatLngs = filteredDevicePoints.map((p) => [p.lat, p.lng]) points.push(...deviceLatLngs) } From 122d5a7d38364c5c0b5b7761693f3ae01c988547 Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 23:17:32 +0100 Subject: [PATCH 5/7] Disable zoom on device Signed-off-by: Arne Hamann --- src/network.js | 2 +- src/views/App.vue | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/network.js b/src/network.js index f9947e34e..2a405e707 100644 --- a/src/network.js +++ b/src/network.js @@ -390,7 +390,7 @@ export function getDevices(myMapId = null) { return axios.get(url, conf) } -export function getDevice(id, myMapId = null) { +export async function getDevice(id, myMapId = null) { const conf = { params: { myMapId, diff --git a/src/views/App.vue b/src/views/App.vue index bab4b0255..314bcb9c9 100644 --- a/src/views/App.vue +++ b/src/views/App.vue @@ -225,7 +225,7 @@ import AppNavigationDevicesItem from '../components/AppNavigationDevicesItem' import AppNavigationMyMapsItem from '../components/AppNavigationMyMapsItem' import optionsController from '../optionsController' import { getLetterColor, hslToRgb, Timer, getDeviceInfoFromUserAgent2, isComputer, isPhone } from '../utils' -import {binSearch, getToken, isPublic} from '../utils/common' +import { binSearch, getToken, isPublic } from '../utils/common' import { poiSearchData } from '../utils/poiData' import { processGpx } from '../tracksUtils' @@ -1975,7 +1975,7 @@ export default { ...device, loading: false, enabled: false, - historyEnabled: false // optionsController.enabledDeviceLines.includes(device.id), + historyEnabled: false, // optionsController.enabledDeviceLines.includes(device.id), } }) this.devices.forEach((device) => { @@ -1993,7 +1993,7 @@ export default { if (device.enabled) { this.disableDevice(device) } else { - this.enableDevice(device, true) + this.enableDevice(device, false) } }, onSearchEnableDevice(device) { @@ -2017,8 +2017,9 @@ export default { }, getDevice(device, enable = false, save = true, zoom = false) { device.loading = true - network.getDevice(device.id, this.myMapId).then((response) => { - this.$set(device, 'points', response.data /*.sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0))*/) + network.getDevice(device.id, this.myMapId).then(async (response) => { + this.$set(device, 'points', response.data /* .sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0)) */) + if (enable) { device.enabled = true } From e7f8f72c663e39e6be87f5ee028496e1d1ea62aa Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Sun, 12 Feb 2023 23:24:31 +0100 Subject: [PATCH 6/7] Use concat to join arrays Signed-off-by: Arne Hamann --- src/components/map/DevicesLayer.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/map/DevicesLayer.vue b/src/components/map/DevicesLayer.vue index b481280cf..da0da1148 100644 --- a/src/components/map/DevicesLayer.vue +++ b/src/components/map/DevicesLayer.vue @@ -98,7 +98,7 @@ export default { ...device.points.slice(firstShownIndex, lastShownIndex + 1), ] const deviceLatLngs = filteredDevicePoints.map((p) => [p.lat, p.lng]) - points.push(...deviceLatLngs) + points = points.concat(deviceLatLngs) } } return points From 901bc2e011320441c321f1aa322cce5284313442 Mon Sep 17 00:00:00 2001 From: Arne Hamann Date: Mon, 13 Feb 2023 19:06:22 +0100 Subject: [PATCH 7/7] Chunck load device point Signed-off-by: Arne Hamann --- lib/Controller/DevicesController.php | 4 ++-- lib/Service/DevicesService.php | 21 ++++++++++++++++++--- src/network.js | 4 +++- src/views/App.vue | 16 ++++++++++++---- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/lib/Controller/DevicesController.php b/lib/Controller/DevicesController.php index 39451d6c0..87864e8c7 100644 --- a/lib/Controller/DevicesController.php +++ b/lib/Controller/DevicesController.php @@ -107,8 +107,8 @@ public function getDevices(): DataResponse { * @param int $pruneBefore * @return DataResponse */ - public function getDevicePoints($id, int $pruneBefore=0): DataResponse { - $points = $this->devicesService->getDevicePointsFromDB($this->userId, $id, $pruneBefore); + public function getDevicePoints($id, ?int $pruneBefore=0, ?int $limit=10000, ?int $offset=0): DataResponse { + $points = $this->devicesService->getDevicePointsFromDB($this->userId, $id, $pruneBefore, $limit, $offset); return new DataResponse($points); } diff --git a/lib/Service/DevicesService.php b/lib/Service/DevicesService.php index 0d2731c31..4cb71dc7f 100644 --- a/lib/Service/DevicesService.php +++ b/lib/Service/DevicesService.php @@ -75,7 +75,16 @@ public function getDevicesFromDB($userId) { return $devices; } - public function getDevicePointsFromDB($userId, $deviceId, $pruneBefore=0) { + /** + * @param $userId + * @param $deviceId + * @param int|null $pruneBefore + * @param int|null $limit + * @param int|null $offset + * @return array + * @throws \OCP\DB\Exception + */ + public function getDevicePointsFromDB($userId, $deviceId, ?int $pruneBefore=0, ?int $limit=null, ?int $offset=null) { $qb = $this->qb; // get coordinates $qb->select('p.id', 'lat', 'lng', 'timestamp', 'altitude', 'accuracy', 'battery') @@ -92,7 +101,13 @@ public function getDevicePointsFromDB($userId, $deviceId, $pruneBefore=0) { $qb->expr()->gt('timestamp', $qb->createNamedParameter($pruneBefore, IQueryBuilder::PARAM_INT)) ); } - $qb->orderBy('timestamp', 'ASC'); + if (!is_null($offset)) { + $qb->setFirstResult($offset); + } + if (!is_null($limit)) { + $qb->setMaxResults($limit); + } + $qb->orderBy('timestamp', 'DESC'); $req = $qb->execute(); $points = []; @@ -110,7 +125,7 @@ public function getDevicePointsFromDB($userId, $deviceId, $pruneBefore=0) { $req->closeCursor(); $qb = $qb->resetQueryParts(); - return $points; + return array_reverse($points); } public function getOrCreateDeviceFromDB($userId, $userAgent) { diff --git a/src/network.js b/src/network.js index 2a405e707..bb6f34048 100644 --- a/src/network.js +++ b/src/network.js @@ -390,10 +390,12 @@ export function getDevices(myMapId = null) { return axios.get(url, conf) } -export async function getDevice(id, myMapId = null) { +export async function getDevice(id, myMapId = null, limit= null, offset = null) { const conf = { params: { myMapId, + limit, + offset, }, } const url = generateUrl('/apps/maps/devices/' + id) diff --git a/src/views/App.vue b/src/views/App.vue index 314bcb9c9..c4872bd08 100644 --- a/src/views/App.vue +++ b/src/views/App.vue @@ -2017,9 +2017,17 @@ export default { }, getDevice(device, enable = false, save = true, zoom = false) { device.loading = true - network.getDevice(device.id, this.myMapId).then(async (response) => { - this.$set(device, 'points', response.data /* .sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0)) */) - + network.getDevice(device.id, this.myMapId, 100000, device.points?.length || 0).then((response) => { + //There are too many points making it responsiv crashes most browsers + // this.$set(device, 'points', response.data /* .sort((p1, p2) => (p1.timestamp || 0) - (p2.timestamp || 0)) */) + if (device.points) { + device.points = response.data.concat(device.points) + } else { + device.points = response.data + } + if (response.data.length >= 100000) { + this.getDevice(device, false, false, false) + } if (enable) { device.enabled = true } @@ -2127,7 +2135,7 @@ export default { // Fixme showInfo('Adding device to map not supported yet') }, - onToggleDeviceHistory(device) { + async onToggleDeviceHistory(device) { device.historyEnabled = !device.historyEnabled this.saveEnabledDeviceLines() },