From c40fbf7dd9f46267aec504a66334c7ccf6779282 Mon Sep 17 00:00:00 2001 From: rusikv Date: Tue, 7 Nov 2023 18:44:46 +0200 Subject: [PATCH 001/304] UI: alarm table widget clear selection on entering edit mode --- .../main/data/json/system/widget_types/alarms_table.json | 2 +- .../widget/lib/alarm/alarms-table-widget.component.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/application/src/main/data/json/system/widget_types/alarms_table.json b/application/src/main/data/json/system/widget_types/alarms_table.json index 5a360bcc328..12716482c99 100644 --- a/application/src/main/data/json/system/widget_types/alarms_table.json +++ b/application/src/main/data/json/system/widget_types/alarms_table.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.alarmsTableWidget.onEditModeChanged();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-alarms-table-widget-settings", diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 20f123972ac..133f484c283 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -321,6 +321,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.ctx.detectChanges(); } + public onEditModeChanged() { + if (this.alarmsDatasource.selection.hasValue()) { + this.alarmsDatasource.clearSelection(); + } + } + public pageLinkSortDirection(): SortDirection { return entityDataPageLinkSortDirection(this.pageLink); } From b75704e6feb3c44fcbd08f2c494398f731ff1682 Mon Sep 17 00:00:00 2001 From: rusikv Date: Wed, 8 Nov 2023 15:50:24 +0200 Subject: [PATCH 002/304] UI: alarm table widget on edit mode if selection has value do not hide title panel with edit action --- .../widget/lib/alarm/alarms-table-widget.component.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 133f484c283..20526591efe 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -322,8 +322,9 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, } public onEditModeChanged() { - if (this.alarmsDatasource.selection.hasValue()) { - this.alarmsDatasource.clearSelection(); + if (this.enableSelection && this.alarmsDatasource.selection.hasValue()) { + this.ctx.hideTitlePanel = !this.ctx.isEdit; + this.ctx.detectChanges(true); } } From acc2b068f514cbe99f16f82cb6e453d20c408f6d Mon Sep 17 00:00:00 2001 From: rusikv Date: Mon, 13 Nov 2023 17:50:13 +0200 Subject: [PATCH 003/304] Added ability to display right layout first in mobile view --- .../dashboard-page.component.html | 4 +- .../dashboard-page.component.ts | 7 ++++ .../dashboard-settings-dialog.component.html | 39 +++++++++++-------- .../dashboard-settings-dialog.component.ts | 7 ++++ ...nage-dashboard-layouts-dialog.component.ts | 3 +- .../src/app/shared/models/dashboard.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 1 + 7 files changed, 42 insertions(+), 20 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index a388f692520..1cbdd2fcf8e 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -296,7 +296,7 @@

{{ translatedDashboardTitle }}

[(opened)]="rightLayoutOpened"> {{ translatedDashboardTitle }} height: mainLayoutSize.height}"> {{settings ? 'dashboard.settings' : 'layout.settings'}} -
+
dashboard.mobile-layout - - {{ 'dashboard.autofill-height' | translate }} + + {{ 'dashboard.display-first-in-mobile-view' | translate }} - - dashboard.mobile-row-height - - - {{ 'dashboard.mobile-row-height-required' | translate }} - - - {{ 'dashboard.min-mobile-row-height-message' | translate }} - - - {{ 'dashboard.max-mobile-row-height-message' | translate }} - - +
+ + {{ 'dashboard.autofill-height' | translate }} + + + dashboard.mobile-row-height + + + {{ 'dashboard.mobile-row-height-required' | translate }} + + + {{ 'dashboard.min-mobile-row-height-message' | translate }} + + + {{ 'dashboard.max-mobile-row-height-message' | translate }} + + +
diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts index 48442113559..303964e0995 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts @@ -32,6 +32,7 @@ import { StatesControllerService } from './states/states-controller.service'; export interface DashboardSettingsDialogData { settings?: DashboardSettings; gridSettings?: GridSettings; + isRightLayout?: boolean; } @Component({ @@ -45,6 +46,7 @@ export class DashboardSettingsDialogComponent extends DialogComponent { if (mobileAutoFillHeightValue) { diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts index 1c04d05bae6..b93a362f921 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts @@ -217,7 +217,8 @@ export class ManageDashboardLayoutsDialogComponent extends DialogComponent { if (data && data.gridSettings) { diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index ffe475d19d2..8b92f9eb397 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -58,6 +58,7 @@ export interface GridSettings { autoFillHeight?: boolean; mobileAutoFillHeight?: boolean; mobileRowHeight?: number; + mobileDisplayLayoutFirst?: boolean; layoutDimension?: LayoutDimension; [key: string]: any; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ef660731bff..194eee430a8 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1113,6 +1113,7 @@ "mobile-row-height-required": "Mobile row height value is required.", "min-mobile-row-height-message": "Only 5 pixels is allowed as minimum mobile row height value.", "max-mobile-row-height-message": "Only 200 pixels is allowed as maximum mobile row height value.", + "display-first-in-mobile-view": "Display first in mobile view", "title-settings": "Title settings", "display-title": "Display dashboard title", "title-color": "Title color", From e5311984bba0d6449e6db65c9fdfac22cf2e98b7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 15:24:40 +0200 Subject: [PATCH 004/304] Housekeeper API --- .../DefaultHousekeeperService.java | 144 ++++++++++++++++++ .../HousekeeperReprocessingService.java | 134 ++++++++++++++++ .../AttributesDeletionTaskProcessor.java | 41 +++++ .../EntityDeletionTaskProcessor.java | 37 +++++ .../EventsDeletionTaskProcessor.java | 38 +++++ .../processor/HousekeeperTaskProcessor.java | 27 ++++ .../TelemetryDeletionTaskProcessor.java | 41 +++++ .../src/main/resources/thingsboard.yml | 7 + common/proto/src/main/proto/queue.proto | 8 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../provider/AwsSqsMonolithQueueFactory.java | 21 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++ .../InMemoryMonolithQueueFactory.java | 20 +++ .../provider/KafkaMonolithQueueFactory.java | 51 ++++++- .../provider/KafkaTbCoreQueueFactory.java | 50 ++++++ .../provider/PubSubMonolithQueueFactory.java | 21 +++ .../provider/PubSubTbCoreQueueFactory.java | 21 +++ .../RabbitMqMonolithQueueFactory.java | 21 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 20 +++ .../ServiceBusMonolithQueueFactory.java | 21 +++ .../ServiceBusTbCoreQueueFactory.java | 21 +++ .../queue/provider/TbCoreQueueFactory.java | 11 ++ .../provider/TbCoreQueueProducerProvider.java | 17 +++ .../provider/TbQueueProducerProvider.java | 6 + .../TbRuleEngineProducerProvider.java | 11 ++ .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 10 ++ .../queue/settings/TbQueueCoreSettings.java | 6 + .../server/dao/device/DeviceServiceImpl.java | 5 + .../dao/entity/AbstractEntityService.java | 6 + .../dao/housekeeper/HousekeeperService.java | 24 +++ .../dao/housekeeper/data/HousekeeperTask.java | 51 +++++++ .../housekeeper/data/HousekeeperTaskType.java | 23 +++ 34 files changed, 951 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java new file mode 100644 index 00000000000..7c7e72b5112 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultHousekeeperService implements HousekeeperService { + + private final Map taskProcessors; + + private final TbQueueConsumer> consumer; + private final TbQueueProducer> producer; + private final HousekeeperReprocessingService reprocessingService; + private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private boolean stopped; + + public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider, + DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.reprocessingService = reprocessingService; + this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); + this.dataDecodingEncodingService = dataDecodingEncodingService; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + + for (TbProtoQueueMsg msg : msgs) { + try { + processTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper service"); + } + + protected void processTask(TbProtoQueueMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + log.error("Unsupported task type {}: {}", task.getTaskType(), task); + return; + } + + log.info("[{}] Processing task: {}", task.getTenantId(), task); + try { + taskProcessor.process(task); + } catch (Exception e) { + log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + reprocessingService.submitForReprocessing(msg); + } + } + + @Override + public void submitTask(HousekeeperTask task) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + .setTask(HousekeeperTaskProto.newBuilder() + .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .build()) + .build()), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper service"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java new file mode 100644 index 00000000000..234cd07c368 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperReprocessingService { + + private final DefaultHousekeeperService housekeeperService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueConsumer> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private final long startTs = System.currentTimeMillis(); + private boolean stopped; + // todo: stats + + public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider) { + this.housekeeperService = housekeeperService; + this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.producerProvider = producerProvider; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + stop(); + return; + } + + for (TbProtoQueueMsg msg : msgs) { + long msgTs = Uuids.unixTimestamp(msg.getKey()); + if (msgTs >= startTs) { + stop(); + return; // fixme: we should commit already reprocessed messages + } + + try { + reprocessTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper tasks reprocessing"); + } + + private void reprocessTask(TbProtoQueueMsg msg) { + housekeeperService.processTask(msg);// fixme: or should we submit to queue? + } + + public void submitForReprocessing(TbProtoQueueMsg msg) { + TbQueueMsgHeaders msgHeaders = msg.getHeaders(); + long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) + .map(header -> bytesToLong(header)) + .orElse(0L); + reprocessingAttempts++; + msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + + var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper tasks reprocessing"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java new file mode 100644 index 00000000000..34bd95a901c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AttributesService attributesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java new file mode 100644 index 00000000000..e1b0877d2ce --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { + + @Override + public void process(HousekeeperTask task) throws Exception { + + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java new file mode 100644 index 00000000000..7b8fdb1242a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; + + @Override + public void process(HousekeeperTask task) throws Exception { + eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_EVENTS; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java new file mode 100644 index 00000000000..bfa7466dc39 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +public interface HousekeeperTaskProcessor { + + void process(HousekeeperTask task) throws Exception; + + HousekeeperTaskType getTaskType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java new file mode 100644 index 00000000000..feb5b0ddc11 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +@Component +@RequiredArgsConstructor +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// timeseriesService.remove() +// timeseriesService.removeAllLatest(); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 70a5146bb7a..17b4c85cfb3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1383,6 +1383,9 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1404,6 +1407,7 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1559,6 +1563,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + topic: "tb_housekeeper" + poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4da2883291b..0dbd96a53dd 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1391,3 +1391,11 @@ message LifecycleEventProto { bool success = 7; string error = 8; } + +message ToHousekeeperServiceMsg { + HousekeeperTaskProto task = 1; +} + +message HousekeeperTaskProto { + bytes value = 1; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index f11eaaef486..9340cafa677 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d23d92ee26d..d3a5eb821d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -43,6 +43,8 @@ public class TbKafkaTopicConfigs { private String fwUpdatesProperties; @Value("${queue.kafka.topic-properties.version-control:}") private String vcProperties; + @Value("${queue.kafka.topic-properties.housekeeper:}") + private String housekeeperProperties; @Getter private Map coreConfigs; @@ -62,6 +64,8 @@ public class TbKafkaTopicConfigs { private Map fwUpdatesConfigs; @Getter private Map vcConfigs; + @Getter + private Map housekeeperConfigs; @PostConstruct private void init() { @@ -76,6 +80,7 @@ private void init() { jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); + housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 8a127384bec..5a61bc1d80e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -226,6 +227,26 @@ public TbQueueProducer(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 6c69b2ec104..a1be25f29d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -205,6 +206,26 @@ public TbQueueProducer(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a681038aa9f..1ee8b404a5b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -160,6 +160,26 @@ public TbQueueProducer(storage, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 662d29af5b3..5ee9b6f53bf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +40,8 @@ import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -82,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -115,6 +117,7 @@ public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafk this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -346,6 +349,52 @@ public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index db6ee521812..c01ba32f705 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -80,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -112,6 +114,7 @@ public KafkaTbCoreQueueFactory(TopicService topicService, this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -303,6 +306,53 @@ public TbQueueProducer> createVersio return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 62fe40f853e..a63a9fc16b1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -227,6 +228,26 @@ public TbQueueProducer(vcAdmin, pubSubSettings, vcSettings.getTopic()); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0e7e1ba2413..5f1f282e390 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1b7f705175e..802b08773d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -224,6 +225,26 @@ public TbQueueProducer(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e0d0ea3b3a6..ceb6e03ca5c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -198,6 +198,26 @@ public TbQueueProducer> createToUsageSta return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 359cbf67b8b..1e9c1cadaf2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -223,6 +224,26 @@ public TbQueueProducer(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index d2ef072ffa2..e39df45b820 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 5aa7866033c..279a469415c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -130,4 +132,13 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * @return */ TbQueueProducer> createVersionControlMsgProducer(); + + TbQueueProducer> createHousekeeperMsgProducer(); + + TbQueueConsumer> createHousekeeperMsgConsumer(); + + TbQueueProducer> createHousekeeperDelayedMsgProducer(); + + TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 5a99f361b44..4fa22cb7e39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -41,6 +43,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; + private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toHousekeeperDelayed; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -55,6 +59,8 @@ public void init() { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); + this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); } @Override @@ -91,4 +97,15 @@ public TbQueueProducer> getTbUsageStatsM public TbQueueProducer> getTbVersionControlMsgProducer() { return toVersionControl; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + return toHousekeeperDelayed; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 4dbdca4375a..5cdf390f476 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -78,4 +79,9 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbVersionControlMsgProducer(); + + TbQueueProducer> getHousekeeperMsgProducer(); + + TbQueueProducer> getHousekeeperDelayedMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e4d98796f4b..c5ddf57fc09 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -89,4 +89,15 @@ public TbQueueProducer> getTbUsageStatsM public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return null; // fixme + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index c83b905a7c2..e8dd7431a65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -85,4 +85,14 @@ public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index dbbaa9f072b..7ddd2109ba6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -81,4 +81,14 @@ public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0a8c12eb8d0..4edb688235d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -34,6 +34,12 @@ public class TbQueueCoreSettings { @Value("${queue.core.usage-stats-topic:tb_usage_stats}") private String usageStatsTopic; + @Value("${queue.core.housekeeper.topic:tb_housekeeper}") + private String housekeeperTopic; + + @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + private String housekeeperDelayedTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f1909231674..f0e5dc1a4a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -328,6 +329,10 @@ private void deleteDevice(TenantId tenantId, Device device) { log.trace("Executing deleteDevice [{}]", device.getId()); deviceCredentialsService.deleteDeviceCredentialsByDeviceId(tenantId, device.getId()); relationService.deleteEntityRelations(tenantId, device.getId()); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, device.getId())); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, device.getId())); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, device.getId())); + // todo: extract to cleanUpRelatedData deviceDao.removeById(tenantId, device.getUuidId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 0baa497cbf7..348093e4fe6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -31,6 +31,9 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.housekeeper.HouseKeeperService; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; import java.util.Collections; @@ -63,6 +66,9 @@ public abstract class AbstractEntityService { @Autowired(required = false) protected EdgeService edgeService; + @Autowired + protected HousekeeperService housekeeperService; + protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); relationService.saveRelation(tenantId, relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java new file mode 100644 index 00000000000..85577a22964 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper; + +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; + +public interface HousekeeperService { + + void submitTask(HousekeeperTask task); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java new file mode 100644 index 00000000000..1f70bebfffb --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper.data; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +/* + * on start, read the retry queue and put the messages back to main queue (save offset) + * */ +@Data +public class HousekeeperTask implements Serializable { + + private final TenantId tenantId; + private final EntityId entityId; + private final HousekeeperTaskType taskType; + + // maybe we should not delete relations asynchronously +// public static HousekeeperTask deleteRelations(TenantId tenantId, EntityId entityId) { +// return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_RELATIONS); +// } + + public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); + } + + public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY); + } + + public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java new file mode 100644 index 00000000000..3f92a2b18f2 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper.data; + +public enum HousekeeperTaskType { + DELETE_ENTITY, + DELETE_ATTRIBUTES, + DELETE_TELEMETRY, // maybe divide into latest and ts kv history? + DELETE_EVENTS +} From 1d5b35a5aa74c75132f1b20bf533deef9f675648 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 16:12:11 +0200 Subject: [PATCH 005/304] Submit cleanup tasks on DeleteEntityEvent --- .../EntityAlarmsDeletionTaskProcessor.java | 40 +++++++++++++++++++ .../server/dao/alarm/AlarmService.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 1 - .../server/dao/asset/BaseAssetService.java | 3 -- .../dao/customer/CustomerServiceImpl.java | 1 - .../dao/dashboard/DashboardServiceImpl.java | 1 - .../dao/device/DeviceProfileServiceImpl.java | 1 - .../server/dao/device/DeviceServiceImpl.java | 8 ---- .../server/dao/edge/EdgeServiceImpl.java | 7 +--- .../dao/entity/AbstractEntityService.java | 31 +++++++++++--- .../dao/entityview/EntityViewServiceImpl.java | 1 - .../dao/housekeeper/data/HousekeeperTask.java | 9 +++++ .../housekeeper/data/HousekeeperTaskType.java | 4 +- .../server/dao/rule/BaseRuleChainService.java | 7 ++-- .../dao/tenant/TenantProfileServiceImpl.java | 1 - .../server/dao/tenant/TenantServiceImpl.java | 1 - .../server/dao/user/UserServiceImpl.java | 1 - 18 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java new file mode 100644 index 00000000000..f9e61fc1cb9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AlarmService alarmService; + + @Override + public void process(HousekeeperTask task) throws Exception { + alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY_ALARMS; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 203fe8e7448..7a55348bed6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -106,7 +106,7 @@ PageData findAlarmDataByQueryForEntities(TenantId tenantId, PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId); + void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 72e1b9c7bf7..a14c172a572 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -189,7 +189,6 @@ public AlarmApiCallResult delAlarm(TenantId tenantId, AlarmId alarmId, boolean c return AlarmApiCallResult.builder().successful(false).build(); } else { var propagationIds = getPropagationEntityIdsList(alarm); - deleteEntityRelations(tenantId, alarm.getId()); alarmDao.removeById(tenantId, alarm.getUuidId()); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId) .entityId(alarmId).entity(alarm).build()); @@ -323,7 +322,7 @@ public AlarmSeverity findHighestAlarmSeverity(TenantId tenantId, EntityId entity } @Override - public void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId) { + public void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteEntityAlarms [{}]", entityId); alarmDao.deleteEntityAlarmRecords(tenantId, entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index a79b70bb7b0..851e68d3dce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,7 +192,6 @@ public void deleteAssetProfile(TenantId tenantId, AssetProfileId assetProfileId) private void removeAssetProfile(TenantId tenantId, AssetProfile assetProfile) { AssetProfileId assetProfileId = assetProfile.getId(); try { - deleteEntityRelations(tenantId, assetProfileId); assetProfileDao.removeById(tenantId, assetProfileId.getId()); publishEvictEvent(new AssetProfileEvictEvent(assetProfile.getTenantId(), assetProfile.getName(), null, assetProfile.getId(), assetProfile.isDefault())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7220a3acdb0..c6550f7e256 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -209,14 +209,11 @@ public void deleteAsset(TenantId tenantId, AssetId assetId) { } Asset asset = assetDao.findById(tenantId, assetId.getId()); - alarmService.deleteEntityAlarmRelations(tenantId, assetId); deleteAsset(tenantId, asset); } private void deleteAsset(TenantId tenantId, Asset asset) { log.trace("Executing deleteAsset [{}]", asset.getId()); - relationService.deleteEntityRelations(tenantId, asset.getId()); - assetDao.removeById(tenantId, asset.getUuidId()); publishEvictEvent(new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), null)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 633333dcbdf..028b6ff59b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -137,7 +137,6 @@ public void deleteCustomer(TenantId tenantId, CustomerId customerId) { deviceService.unassignCustomerDevices(customer.getTenantId(), customerId); edgeService.unassignCustomerEdges(customer.getTenantId(), customerId); userService.deleteCustomerUsers(customer.getTenantId(), customerId); - deleteEntityRelations(tenantId, customerId); apiUsageStateService.deleteApiUsageStateByEntityId(customerId); customerDao.removeById(tenantId, customerId.getId()); countService.publishCountEntityEvictEvent(tenantId, EntityType.CUSTOMER); diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index b60f356b0b5..e08653856b5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -234,7 +234,6 @@ private Dashboard updateAssignedCustomer(TenantId tenantId, DashboardId dashboar public void deleteDashboard(TenantId tenantId, DashboardId dashboardId) { log.trace("Executing deleteDashboard [{}]", dashboardId); Validator.validateId(dashboardId, INCORRECT_DASHBOARD_ID + dashboardId); - deleteEntityRelations(tenantId, dashboardId); try { dashboardDao.removeById(tenantId, dashboardId.getId()); publishEvictEvent(new DashboardTitleEvictEvent(dashboardId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 403df3e822a..f9be1c9988a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -235,7 +235,6 @@ public void deleteDeviceProfile(TenantId tenantId, DeviceProfileId deviceProfile private void removeDeviceProfile(TenantId tenantId, DeviceProfile deviceProfile) { DeviceProfileId deviceProfileId = deviceProfile.getId(); try { - deleteEntityRelations(tenantId, deviceProfileId); deviceProfileDao.removeById(tenantId, deviceProfileId.getId()); publishEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), null, deviceProfile.getId(), deviceProfile.isDefault(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f0e5dc1a4a6..01894a8cb89 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -75,7 +75,6 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -321,19 +320,12 @@ public void deleteDevice(final TenantId tenantId, final DeviceId deviceId) { } Device device = deviceDao.findById(tenantId, deviceId.getId()); - alarmService.deleteEntityAlarmRelations(tenantId, deviceId); deleteDevice(tenantId, device); } private void deleteDevice(TenantId tenantId, Device device) { log.trace("Executing deleteDevice [{}]", device.getId()); deviceCredentialsService.deleteDeviceCredentialsByDeviceId(tenantId, device.getId()); - relationService.deleteEntityRelations(tenantId, device.getId()); - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, device.getId())); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, device.getId())); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, device.getId())); - // todo: extract to cleanUpRelatedData - deviceDao.removeById(tenantId, device.getUuidId()); DeviceCacheEvictEvent deviceCacheEvictEvent = new DeviceCacheEvictEvent(device.getTenantId(), device.getId(), device.getName(), null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 80eea0a478d..118449b6981 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; @@ -49,7 +48,6 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -58,6 +56,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -212,12 +211,10 @@ public void deleteEdge(TenantId tenantId, EdgeId edgeId) { validateId(edgeId, INCORRECT_EDGE_ID + edgeId); Edge edge = edgeDao.findById(tenantId, edgeId.getId()); - - deleteEntityRelations(tenantId, edgeId); - edgeDao.removeById(tenantId, edgeId.getId()); publishEvictEvent(new EdgeCacheEvictEvent(edge.getTenantId(), edge.getName(), null)); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(edgeId).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 348093e4fe6..ba37737b84e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EdgeId; @@ -30,8 +32,8 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.housekeeper.HouseKeeperService; import org.thingsboard.server.dao.housekeeper.HousekeeperService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; @@ -69,6 +71,28 @@ public abstract class AbstractEntityService { @Autowired protected HousekeeperService housekeeperService; + @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService + public void onEntityDeleted(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); + + cleanUpRelatedData(tenantId, entityId); + if (EntityType.USER.equals(entityId.getEntityType())) { +// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); +// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + } + } + + protected void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + // todo: skipped entities list + relationService.deleteEntityRelations(tenantId, entityId); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + } + protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); relationService.saveRelation(tenantId, relation); @@ -79,11 +103,6 @@ protected void deleteRelation(TenantId tenantId, EntityRelation relation) { relationService.deleteRelation(tenantId, relation); } - protected void deleteEntityRelations(TenantId tenantId, EntityId entityId) { - relationService.deleteEntityRelations(tenantId, entityId); - alarmService.deleteEntityAlarmRelations(tenantId, entityId); - } - protected static Optional extractConstraintViolationException(Exception t) { if (t instanceof ConstraintViolationException) { return Optional.of((ConstraintViolationException) t); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index ed962ce84c1..6940b427944 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -331,7 +331,6 @@ public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) public void deleteEntityView(TenantId tenantId, EntityViewId entityViewId) { log.trace("Executing deleteEntityView [{}]", entityViewId); validateId(entityViewId, INCORRECT_ENTITY_VIEW_ID + entityViewId); - deleteEntityRelations(tenantId, entityViewId); EntityView entityView = entityViewDao.findById(tenantId, entityViewId.getId()); entityViewDao.removeById(tenantId, entityViewId.getId()); publishEvictEvent(new EntityViewEvictEvent(entityView.getTenantId(), entityView.getId(), entityView.getEntityId(), null, entityView.getName(), null)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 1f70bebfffb..7ab415fe7b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -18,6 +18,7 @@ import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import java.io.Serializable; @@ -48,4 +49,12 @@ public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } + public static HousekeeperTask unassignAlarms(TenantId tenantId, UserId userId) { + return new HousekeeperTask(tenantId, userId, HousekeeperTaskType.UNASSIGN_ALARMS); + } + + public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ENTITY_ALARMS); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java index 3f92a2b18f2..22951280c18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java @@ -19,5 +19,7 @@ public enum HousekeeperTaskType { DELETE_ENTITY, DELETE_ATTRIBUTES, DELETE_TELEMETRY, // maybe divide into latest and ts kv history? - DELETE_EVENTS + DELETE_EVENTS, + UNASSIGN_ALARMS, + DELETE_ENTITY_ALARMS } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1d1296c42ef..88fa6f1351b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -177,7 +177,7 @@ public RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainM List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - deleteEntityRelations(tenantId, existingNode.getId()); + cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -771,7 +771,7 @@ private void checkRuleNodesAndDelete(TenantId tenantId, RuleChainId ruleChainId) private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); for (var node : ruleNodes) { - deleteEntityRelations(tenantId, node.getId()); + cleanUpRelatedData(tenantId, node.getId()); } ruleNodeDao.deleteByIdIn(ruleNodeIds); } @@ -783,7 +783,6 @@ public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); } - deleteEntityRelations(tenantId, ruleChainId); } @@ -820,8 +819,8 @@ private List getNodeToRuleChainRelations(TenantId tenantId, Rule } private void deleteRuleNode(TenantId tenantId, EntityId entityId) { - deleteEntityRelations(tenantId, entityId); ruleNodeDao.removeById(tenantId, entityId.getId()); + cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 31a419a5d00..2462ca19d14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -130,7 +130,6 @@ private void removeTenantProfile(TenantId tenantId, TenantProfileId tenantProfil throw t; } } - deleteEntityRelations(tenantId, tenantProfileId); publishEvictEvent(new TenantProfileEvictEvent(tenantProfileId, isDefault)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(tenantProfileId).build()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index d91f469c1bd..6fb2a93d544 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -245,7 +245,6 @@ public void deleteTenant(TenantId tenantId) { tenantDao.removeById(tenantId, tenantId.getId()); publishEvictEvent(new TenantEvictEvent(tenantId, true)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(tenantId).build()); - relationService.deleteEntityRelations(tenantId, tenantId); alarmService.deleteEntityAlarmRecordsByTenantId(tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 17965408539..7f374b72591 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -255,7 +255,6 @@ public void deleteUser(TenantId tenantId, User user) { validateId(userId, INCORRECT_USER_ID + userId); userCredentialsDao.removeByUserId(tenantId, userId); userAuthSettingsDao.removeByUserId(userId); - deleteEntityRelations(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); From d5d18f11380c1f462481442c7ea4c543adfc11c5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 16:50:03 +0200 Subject: [PATCH 006/304] Alarms unassign housekeeper task; remove in-memory housekeeper service --- .../entitiy/alarm/DefaultTbAlarmService.java | 16 +-- .../service/entitiy/alarm/TbAlarmService.java | 2 +- .../InMemoryHouseKeeperServiceService.java | 103 ------------------ .../AlarmsUnassignTaskProcessor.java | 41 +++++++ .../AttributesDeletionTaskProcessor.java | 3 +- .../EntityAlarmsDeletionTaskProcessor.java | 2 +- .../EntityDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 2 +- .../processor/HousekeeperTaskProcessor.java | 4 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 2 +- .../dao/entity/AbstractEntityService.java | 6 +- .../data/AlarmsUnassignHousekeeperTask.java | 18 +-- .../dao/housekeeper/data/HousekeeperTask.java | 19 ++-- 14 files changed, 82 insertions(+), 140 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java rename common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java => dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java (59%) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 5305018b84c..deefdaa6143 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -176,16 +176,16 @@ public AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws Things } @Override - public List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { + public List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { List totalAlarmIds = new ArrayList<>(); PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); while (true) { - PageData pageData = alarmService.findAlarmIdsByAssigneeId(user.getTenantId(), user.getId(), pageLink); + PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); List alarmIds = pageData.getData(); if (alarmIds.isEmpty()) { break; } - processAlarmsUnassignment(tenantId, user, alarmIds, unassignTs); + processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs); totalAlarmIds.addAll(alarmIds); pageLink = pageLink.nextPageLink(); } @@ -204,16 +204,16 @@ private static long getOrDefault(long ts) { return ts > 0 ? ts : System.currentTimeMillis(); } - private void processAlarmsUnassignment(TenantId tenantId, User user, List alarmIds, long unassignTs) { + private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List alarmIds, long unassignTs) { for (AlarmId alarmId : alarmIds) { - log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, user.getId()); - AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(user.getTenantId(), alarmId, unassignTs); + log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId); + AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs); if (!result.isSuccessful()) { - log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, user.getId()); + log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId); continue; } if (result.isModified()) { - String comment = String.format("Alarm was unassigned because user %s - was deleted", user.getTitle()); + String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); notificationEntityService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 34dc3d11f21..c975291b581 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -41,7 +41,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java deleted file mode 100644 index 48709906068..00000000000 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.housekeeper; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.HouseKeeperService; -import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@RequiredArgsConstructor -@Slf4j -public class InMemoryHouseKeeperServiceService implements HouseKeeperService { - - final TbAlarmService alarmService; - - ListeningExecutorService executor; - - AtomicInteger queueSize = new AtomicInteger(); - AtomicInteger totalProcessedCounter = new AtomicInteger(); - - @PostConstruct - public void init() { - log.debug("Starting HouseKeeper service"); - executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper"))); - } - - @PreDestroy - public void destroy() { - if (executor != null) { - log.debug("Stopping HouseKeeper service"); - executor.shutdown(); - } - } - - @TransactionalEventListener(fallbackExecution = true) - public void handleEvent(DeleteEntityEvent event) { - log.trace("[{}] DeleteEntityEvent handler: {}", event.getTenantId(), event); - EntityId entityId = event.getEntityId(); - if (EntityType.USER.equals(entityId.getEntityType())) { - unassignDeletedUserAlarms(event.getTenantId(), (User) event.getEntity(), event.getTs()); - } - } - - @Override - public ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { - log.debug("[{}][{}] unassignDeletedUserAlarms submitting, pending queue size: {} ", tenantId, user.getId().getId(), queueSize.get()); - queueSize.incrementAndGet(); - ListenableFuture> future = executor.submit(() -> alarmService.unassignDeletedUserAlarms(tenantId, user, unassignTs)); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(List alarmIds) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.debug("[{}][{}] unassignDeletedUserAlarms finished, pending queue size: {}, total processed count: {} ", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get()); - } - - @Override - public void onFailure(Throwable throwable) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.error("[{}][{}] unassignDeletedUserAlarms failed, pending queue size: {}, total processed count: {}", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get(), throwable); - } - }, MoreExecutors.directExecutor()); - return future; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java new file mode 100644 index 00000000000..54bf7989afa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; + +@Component +@RequiredArgsConstructor +public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { + + private final TbAlarmService alarmService; + + @Override + public void process(AlarmsUnassignHousekeeperTask task) throws Exception { + alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.UNASSIGN_ALARMS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 34bd95a901c..169ac3e5730 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -17,14 +17,13 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AttributesService attributesService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java index f9e61fc1cb9..a91f2262f98 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ @Component @RequiredArgsConstructor -public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AlarmService alarmService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java index e1b0877d2ce..deb693c466f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -22,7 +22,7 @@ @Component @RequiredArgsConstructor -public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { @Override public void process(HousekeeperTask task) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 7b8fdb1242a..56edf016371 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ @Component @RequiredArgsConstructor -public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final EventService eventService; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index bfa7466dc39..7daa99540e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -18,9 +18,9 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; -public interface HousekeeperTaskProcessor { +public interface HousekeeperTaskProcessor { - void process(HousekeeperTask task) throws Exception; + void process(T task) throws Exception; HousekeeperTaskType getTaskType(); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index feb5b0ddc11..2a491148ea5 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -23,7 +23,7 @@ @Component @RequiredArgsConstructor -public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 5d493193db0..38b64d50ed2 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -173,7 +173,7 @@ public void testUnassignDeletedUserAlarms() throws ThingsboardException { User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user, System.currentTimeMillis()); + service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index ba37737b84e..7bfe76e753e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -78,9 +79,8 @@ public void onEntityDeleted(DeleteEntityEvent event) { log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); - if (EntityType.USER.equals(entityId.getEntityType())) { -// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); -// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + if (entityId.getEntityType() == EntityType.USER) { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java similarity index 59% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java rename to dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index 55b164df526..876a8f736b8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -13,17 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper; +package org.thingsboard.server.dao.housekeeper.data; -import com.google.common.util.concurrent.ListenableFuture; +import lombok.Getter; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.TenantId; -import java.util.List; +@Getter +public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { -public interface HouseKeeperService { + private final String userTitle; + private final long ts; - ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + protected AlarmsUnassignHousekeeperTask(User user) { + super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); + this.userTitle = user.getTitle(); + this.ts = System.currentTimeMillis(); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 7ab415fe7b7..726e07a790d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,27 +15,28 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import java.io.Serializable; /* * on start, read the retry queue and put the messages back to main queue (save offset) * */ -@Data +@Getter public class HousekeeperTask implements Serializable { private final TenantId tenantId; private final EntityId entityId; private final HousekeeperTaskType taskType; - // maybe we should not delete relations asynchronously -// public static HousekeeperTask deleteRelations(TenantId tenantId, EntityId entityId) { -// return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_RELATIONS); -// } + protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { + this.tenantId = tenantId; + this.entityId = entityId; + this.taskType = taskType; + } public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); @@ -49,8 +50,8 @@ public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } - public static HousekeeperTask unassignAlarms(TenantId tenantId, UserId userId) { - return new HousekeeperTask(tenantId, userId, HousekeeperTaskType.UNASSIGN_ALARMS); + public static HousekeeperTask unassignAlarms(User user) { + return new AlarmsUnassignHousekeeperTask(user); } public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId entityId) { From 7e07771accb561d487a57f28f6e72b8e2a8437b1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 14 Feb 2024 20:20:08 +0200 Subject: [PATCH 007/304] Housekeeper tasks reprocessing --- .../DefaultHousekeeperService.java | 42 ++++++++----- .../HousekeeperReprocessingService.java | 50 ++++++++-------- .../AlarmsUnassignTaskProcessor.java | 8 ++- .../TelemetryDeletionTaskProcessor.java | 2 +- .../src/main/resources/thingsboard.yml | 1 + common/proto/src/main/proto/queue.proto | 2 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/settings/TbQueueCoreSettings.java | 2 +- .../dao/entity/AbstractEntityService.java | 31 +--------- .../dao/housekeeper/CleanUpService.java | 60 +++++++++++++++++++ .../data/AlarmsUnassignHousekeeperTask.java | 2 - .../dao/housekeeper/data/HousekeeperTask.java | 9 ++- .../server/dao/rule/BaseRuleChainService.java | 6 +- .../dao/sql/attributes/JpaAttributeDao.java | 21 +++++-- 14 files changed, 153 insertions(+), 87 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 7c7e72b5112..f669df53054 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.housekeeper; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -40,6 +40,8 @@ import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -49,7 +51,7 @@ @Slf4j public class DefaultHousekeeperService implements HousekeeperService { - private final Map taskProcessors; + private final Map> taskProcessors; private final TbQueueConsumer> consumer; private final TbQueueProducer> producer; @@ -65,7 +67,8 @@ public class DefaultHousekeeperService implements HousekeeperService { public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, - DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + DataDecodingEncodingService dataDecodingEncodingService, + @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; @@ -86,7 +89,7 @@ public void afterStartUp() { for (TbProtoQueueMsg msg : msgs) { try { - processTask(msg); + processTask(msg.getValue()); } catch (Exception e) { log.error("Message processing failed", e); } @@ -107,19 +110,18 @@ public void afterStartUp() { log.info("Started Housekeeper service"); } - protected void processTask(TbProtoQueueMsg msg) { - HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); - HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); - if (taskProcessor == null) { - log.error("Unsupported task type {}: {}", task.getTaskType(), task); - return; - } + @SuppressWarnings("unchecked") + protected void processTask(ToHousekeeperServiceMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); - log.info("[{}] Processing task: {}", task.getTenantId(), task); + log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); try { - taskProcessor.process(task); + taskProcessor.process((T) task); } catch (Exception e) { - log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", + task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + task.getTaskType(), msg.getTask().getAttempt(), task, e); reprocessingService.submitForReprocessing(msg); } } @@ -127,11 +129,20 @@ protected void processTask(TbProtoQueueMsg msg) { @Override public void submitTask(HousekeeperTask task) { TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .setTs(task.getTs()) + .setAttempt(0) .build()) .build()), null); + log.trace("[{}][{}][{}] Submitted task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + } + + @SuppressWarnings("unchecked") + private HousekeeperTaskProcessor getTaskProcessor(HousekeeperTaskType taskType) { + return Optional.ofNullable((HousekeeperTaskProcessor) taskProcessors.get(taskType)) + .orElseThrow(() -> new IllegalArgumentException("Unsupported task type " + taskType)); } @PreDestroy @@ -141,4 +152,5 @@ private void stop() { consumer.unsubscribe(); consumerExecutor.shutdownNow(); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 234cd07c368..4fca14e44f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -15,16 +15,15 @@ */ package org.thingsboard.server.service.housekeeper; -import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -33,13 +32,10 @@ import javax.annotation.PreDestroy; import java.util.List; -import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; - @TbCoreComponent @Service @Slf4j @@ -77,15 +73,21 @@ public void afterStartUp() { return; } - for (TbProtoQueueMsg msg : msgs) { - long msgTs = Uuids.unixTimestamp(msg.getKey()); - if (msgTs >= startTs) { - stop(); - return; // fixme: we should commit already reprocessed messages + if (msgs.stream().anyMatch(msg -> { + boolean newMsg = msg.getValue().getTask().getTs() >= startTs; + if (newMsg) { + log.info("Stopping reprocessing due to msg is new {}", msg); } - + return newMsg; + })) { + stop(); // fixme: we should commit already reprocessed messages; maybe submit for reprocessing again and commit? + // msg batch size should be 1. otherwise some tasks won't be reprocessed + return; + } + for (TbProtoQueueMsg msg : msgs) { try { - reprocessTask(msg); + housekeeperService.processTask(msg.getValue());// fixme: or should we submit to queue? + Thread.sleep(1000); } catch (Exception e) { log.error("Message processing failed", e); } @@ -106,21 +108,19 @@ public void afterStartUp() { log.info("Started Housekeeper tasks reprocessing"); } - private void reprocessTask(TbProtoQueueMsg msg) { - housekeeperService.processTask(msg);// fixme: or should we submit to queue? - } - - public void submitForReprocessing(TbProtoQueueMsg msg) { - TbQueueMsgHeaders msgHeaders = msg.getHeaders(); - long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) - .map(header -> bytesToLong(header)) - .orElse(0L); - reprocessingAttempts++; - msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + public void submitForReprocessing(ToHousekeeperServiceMsg msg) { + HousekeeperTaskProto task = msg.getTask(); + int attempt = task.getAttempt() + 1; + msg = msg.toBuilder() + .setTask(task.toBuilder() + .setAttempt(attempt) + .setTs(System.currentTimeMillis()) + .build()) + .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 54bf7989afa..247466af65e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -16,21 +16,27 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; +import java.util.List; + @Component @RequiredArgsConstructor +@Slf4j public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { private final TbAlarmService alarmService; @Override public void process(AlarmsUnassignHousekeeperTask task) throws Exception { - alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + List alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index 2a491148ea5..e925c9f20c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -35,7 +35,7 @@ public void process(HousekeeperTask task) throws Exception { @Override public HousekeeperTaskType getTaskType() { - return HousekeeperTaskType.DELETE_ATTRIBUTES; + return HousekeeperTaskType.DELETE_TELEMETRY; } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 17b4c85cfb3..6a96c0a3dbb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1565,6 +1565,7 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: topic: "tb_housekeeper" + reprocessing-topic: "tb_housekeeper.reprocessing" poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 0dbd96a53dd..c4e8e64e422 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1398,4 +1398,6 @@ message ToHousekeeperServiceMsg { message HousekeeperTaskProto { bytes value = 1; + int64 ts = 2; + int32 attempt = 3; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 9340cafa677..f11eaaef486 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - public static byte[] longToBytes(long x) { + protected static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - public static long bytesToLong(byte[] bytes) { + protected static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 4edb688235d..0b66c2fa75a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -37,7 +37,7 @@ public class TbQueueCoreSettings { @Value("${queue.core.housekeeper.topic:tb_housekeeper}") private String housekeeperTopic; - @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}") private String housekeeperDelayedTopic; @Value("${queue.core.partitions}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 7bfe76e753e..6b88f196991 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -20,11 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -33,10 +30,8 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.housekeeper.HousekeeperService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.CleanUpService; import org.thingsboard.server.dao.relation.RelationService; import java.util.Collections; @@ -70,28 +65,8 @@ public abstract class AbstractEntityService { protected EdgeService edgeService; @Autowired - protected HousekeeperService housekeeperService; - - @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService - public void onEntityDeleted(DeleteEntityEvent event) { - TenantId tenantId = event.getTenantId(); - EntityId entityId = event.getEntityId(); - log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); - - cleanUpRelatedData(tenantId, entityId); - if (entityId.getEntityType() == EntityType.USER) { - housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); - } - } - - protected void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { - // todo: skipped entities list - relationService.deleteEntityRelations(tenantId, entityId); - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); - } + @Lazy + protected CleanUpService cleanUpService; protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java new file mode 100644 index 00000000000..0086e179268 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.relation.RelationService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CleanUpService { + + private final HousekeeperService housekeeperService; + private final RelationService relationService; + + @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService + public void handleEntityDeletionEvent(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); + + log.info("[{}][{}][{}] Handling DeleteEntityEvent", tenantId, entityId.getEntityType(), entityId.getId()); + cleanUpRelatedData(tenantId, entityId); + if (entityId.getEntityType() == EntityType.USER) { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + } + } + + public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + // todo: skipped entities list + relationService.deleteEntityRelations(tenantId, entityId); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index 876a8f736b8..aa1858f59fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -22,12 +22,10 @@ public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { private final String userTitle; - private final long ts; protected AlarmsUnassignHousekeeperTask(User user) { super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); this.userTitle = user.getTitle(); - this.ts = System.currentTimeMillis(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 726e07a790d..9fcbbfe43d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,27 +15,26 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import java.io.Serializable; -/* - * on start, read the retry queue and put the messages back to main queue (save offset) - * */ -@Getter +@Data public class HousekeeperTask implements Serializable { private final TenantId tenantId; private final EntityId entityId; private final HousekeeperTaskType taskType; + private final long ts; protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { this.tenantId = tenantId; this.entityId = entityId; this.taskType = taskType; + this.ts = System.currentTimeMillis(); } public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 88fa6f1351b..ebbec8969f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -177,7 +177,7 @@ public RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainM List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? + cleanUpService.cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -771,7 +771,7 @@ private void checkRuleNodesAndDelete(TenantId tenantId, RuleChainId ruleChainId) private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); for (var node : ruleNodes) { - cleanUpRelatedData(tenantId, node.getId()); + cleanUpService.cleanUpRelatedData(tenantId, node.getId()); } ruleNodeDao.deleteByIdIn(ruleNodeIds); } @@ -820,7 +820,7 @@ private List getNodeToRuleChainRelations(TenantId tenantId, Rule private void deleteRuleNode(TenantId tenantId, EntityId entityId) { ruleNodeDao.removeById(tenantId, entityId.getId()); - cleanUpRelatedData(tenantId, entityId); + cleanUpService.cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index cb51a7d8adb..d8cf6aab329 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -20,9 +20,11 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -31,6 +33,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; @@ -132,10 +135,10 @@ public List find(TenantId tenantId, EntityId entityId, String @Override public List findAll(TenantId tenantId, EntityId entityId, String attributeType) { return DaoUtil.convertDataList(Lists.newArrayList( - attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( - entityId.getEntityType(), - entityId.getId(), - attributeType))); + attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( + entityId.getEntityType(), + entityId.getId(), + attributeType))); } @Override @@ -188,6 +191,16 @@ public List> removeAll(TenantId tenantId, EntityId enti return futuresList; } + @Transactional + @Override + public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_type = ? and entity_id = ? " + + "RETURNING attribute_type, attribute_key", entityId.getEntityType().name(), entityId.getId()).stream() + .map(deleted -> Pair.of((String) deleted.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), + (String) deleted.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) + .collect(Collectors.toList()); + } + private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, String attributeType, String attributeKey) { return new AttributeKvCompositeKey( entityId.getEntityType(), From a1bede3cbb4ab914b21b05c37b51afd0006809a8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 14 Feb 2024 20:20:26 +0200 Subject: [PATCH 008/304] Attributes deletion task processor --- .../AttributesDeletionTaskProcessor.java | 5 +- .../dao/attributes/AttributesService.java | 2 + .../server/dao/attributes/AttributesDao.java | 4 + .../dao/attributes/BaseAttributesService.java | 8 ++ .../attributes/CachedAttributesService.java | 94 +++++++++++-------- 5 files changed, 71 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 169ac3e5730..e3bac35e9b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; @@ -23,13 +24,15 @@ @Component @RequiredArgsConstructor +@Slf4j public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AttributesService attributesService; @Override public void process(HousekeeperTask task) throws Exception { -// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); + log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 3dd96267397..4f0840c7041 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -49,4 +49,6 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds, String scope); + int removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 08a9419885d..8cfc6be57b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -46,4 +47,7 @@ public interface AttributesDao { List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, EntityType entityType, List entityIds, String attributeType); + + List> removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 342cb264850..1cace5b71cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -18,6 +18,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -113,4 +114,11 @@ public ListenableFuture> removeAll(TenantId tenantId, EntityId enti validate(entityId, scope); return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } + + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + return deleted.size(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 1666015a486..1751d33312f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -147,47 +148,47 @@ public ListenableFuture> find(TenantId tenantId, EntityId return Futures.transformAsync(cacheExecutor.submit(() -> findCachedAttributes(entityId, scope, attributeKeys)), wrappedCachedAttributes -> { - List cachedAttributes = wrappedCachedAttributes.values().stream() - .map(TbCacheValueWrapper::get) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - if (wrappedCachedAttributes.size() == attributeKeys.size()) { - log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); - return Futures.immediateFuture(cachedAttributes); - } - - Set notFoundAttributeKeys = new HashSet<>(attributeKeys); - notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - - List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - - // DB call should run in DB executor, not in cache-related executor - return jpaExecutorService.submit(() -> { - var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); - try { - log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - for (AttributeKvEntry foundInDbAttribute : result) { - AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); - cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); - notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); - } - for (String key : notFoundAttributeKeys) { - cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); - } - List mergedAttributes = new ArrayList<>(cachedAttributes); - mergedAttributes.addAll(result); - cacheTransaction.commit(); - log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); - return mergedAttributes; - } catch (Throwable e) { - cacheTransaction.rollback(); - log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); - throw e; - } - }); - - }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor + List cachedAttributes = wrappedCachedAttributes.values().stream() + .map(TbCacheValueWrapper::get) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (wrappedCachedAttributes.size() == attributeKeys.size()) { + log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); + return Futures.immediateFuture(cachedAttributes); + } + + Set notFoundAttributeKeys = new HashSet<>(attributeKeys); + notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); + + List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); + + // DB call should run in DB executor, not in cache-related executor + return jpaExecutorService.submit(() -> { + var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); + try { + log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + for (AttributeKvEntry foundInDbAttribute : result) { + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); + cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); + } + for (String key : notFoundAttributeKeys) { + cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(result); + cacheTransaction.commit(); + log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); + return mergedAttributes; + } catch (Throwable e) { + cacheTransaction.rollback(); + log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); + throw e; + } + }); + + }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor } private Map> findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { @@ -268,4 +269,15 @@ public ListenableFuture> removeAll(TenantId tenantId, EntityId enti }, cacheExecutor)).collect(Collectors.toList())); } + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> result = attributesDao.removeAllByEntityId(tenantId, entityId); + result.forEach(deleted -> { + String scope = deleted.getKey(); + String key = deleted.getValue(); + cache.evict(new AttributeCacheKey(scope, entityId, key)); + }); + return result.size(); + } + } From faa93de1027bacffe781611b5dfbc60f25b268c5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 15 Feb 2024 14:26:33 +0200 Subject: [PATCH 009/304] Tenant deletion with housekeeper --- .../DefaultHousekeeperService.java | 7 +++- .../HousekeeperReprocessingService.java | 5 ++- ...ava => EntitiesDeletionTaskProcessor.java} | 18 +++++--- .../server/dao/entity/EntityDaoService.java | 4 ++ .../server/common/data/EntityType.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 5 +++ .../server/dao/asset/BaseAssetService.java | 5 +++ .../dao/customer/CustomerServiceImpl.java | 5 +++ .../dao/dashboard/DashboardServiceImpl.java | 5 +++ .../dao/device/DeviceProfileServiceImpl.java | 5 +++ .../server/dao/device/DeviceServiceImpl.java | 6 +++ .../server/dao/edge/EdgeServiceImpl.java | 5 +++ .../dao/entityview/EntityViewServiceImpl.java | 5 +++ .../dao/housekeeper/CleanUpService.java | 9 ++++ .../dao/housekeeper/HousekeeperService.java | 5 +++ .../data/EntitiesDeletionHousekeeperTask.java | 32 +++++++++++++++ .../dao/housekeeper/data/HousekeeperTask.java | 5 +++ .../housekeeper/data/HousekeeperTaskType.java | 2 +- .../DefaultNotificationRequestService.java | 5 +++ .../DefaultNotificationRuleService.java | 5 +++ .../DefaultNotificationTargetService.java | 5 +++ .../DefaultNotificationTemplateService.java | 5 +++ .../server/dao/ota/BaseOtaPackageService.java | 5 +++ .../server/dao/queue/BaseQueueService.java | 5 +++ .../dao/resource/BaseResourceService.java | 5 +++ .../server/dao/rpc/BaseRpcService.java | 5 +++ .../server/dao/rule/BaseRuleChainService.java | 5 +++ .../settings/AdminSettingsServiceImpl.java | 23 ++++++++++- .../dao/sql/attributes/JpaAttributeDao.java | 4 +- .../dao/sql/user/JpaUserSettingsDao.java | 6 +++ .../dao/sql/user/UserSettingsRepository.java | 11 +++++ .../server/dao/tenant/TenantServiceImpl.java | 41 ++++++------------- .../usagerecord/ApiUsageStateServiceImpl.java | 5 +++ .../server/dao/user/UserServiceImpl.java | 20 +++++++++ .../server/dao/user/UserSettingsDao.java | 3 ++ .../dao/widget/WidgetTypeServiceImpl.java | 5 +++ .../dao/widget/WidgetsBundleServiceImpl.java | 5 +++ 37 files changed, 258 insertions(+), 41 deletions(-) rename application/src/main/java/org/thingsboard/server/service/housekeeper/processor/{EntityDeletionTaskProcessor.java => EntitiesDeletionTaskProcessor.java} (53%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index f669df53054..9361baae2b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -128,8 +128,13 @@ protected void processTask(ToHousekeeperServiceMsg m @Override public void submitTask(HousekeeperTask task) { + submitTask(UUID.randomUUID(), task); + } + + @Override + public void submitTask(UUID key, HousekeeperTask task) { TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ToHousekeeperServiceMsg.newBuilder() + producer.send(tpi, new TbProtoQueueMsg<>(key, ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) .setTs(task.getTs()) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 4fca14e44f2..ac0051def13 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -49,7 +49,7 @@ public class HousekeeperReprocessingService { @Value("${queue.core.housekeeper.poll-interval-ms:10000}") private int pollInterval; - private final long startTs = System.currentTimeMillis(); + private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing private boolean stopped; // todo: stats @@ -114,12 +114,13 @@ public void submitForReprocessing(ToHousekeeperServiceMsg msg) { msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis()) + .setTs(System.currentTimeMillis()) // maybe set ts + 1 hour so that no-one reprocesses it immediately .build()) .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + // fixme submit with the same msg key, so that the messages goes to this consumer and will not be processed by anyone else producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java rename to application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index deb693c466f..da7af3c42be 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -17,21 +17,29 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.dao.housekeeper.data.EntitiesDeletionHousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntitiesDeletionTaskProcessor implements HousekeeperTaskProcessor { - @Override - public void process(HousekeeperTask task) throws Exception { + private final EntityServiceRegistry entityServiceRegistry; + @Override + public void process(EntitiesDeletionHousekeeperTask task) throws Exception { + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(task.getEntityType()); + if (entityService == null) { + throw new IllegalArgumentException("Unsupported entity type " + task.getEntityType()); + } + entityService.deleteByTenantId(task.getTenantId()); } @Override public HousekeeperTaskType getTaskType() { - return HousekeeperTaskType.DELETE_ENTITY; + return HousekeeperTaskType.DELETE_ENTITIES; } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java index b1f6ab9cdbe..992c4fad943 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java @@ -34,6 +34,10 @@ default void deleteEntity(TenantId tenantId, EntityId id) { throw new IllegalArgumentException(getEntityType().getNormalName() + " deletion not supported"); } + default void deleteByTenantId(TenantId tenantId) { + throw new IllegalArgumentException("Deletion by tenant id not supported for " + getEntityType().getNormalName()); + } + EntityType getEntityType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index fb4fe1011e5..6316ab12819 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -58,7 +58,8 @@ public String getNormalName () { NOTIFICATION_TEMPLATE (30), NOTIFICATION_REQUEST (31), NOTIFICATION (32), - NOTIFICATION_RULE (33); + NOTIFICATION_RULE (33), + ADMIN_SETTINGS(34); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 851e68d3dce..686b75befbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -304,6 +304,11 @@ public void deleteAssetProfilesByTenantId(TenantId tenantId) { tenantAssetProfilesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAssetProfilesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findAssetProfileById(tenantId, new AssetProfileId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index c6550f7e256..a57b232ec2d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -279,6 +279,11 @@ public void deleteAssetsByTenantId(TenantId tenantId) { tenantAssetsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAssetsByTenantId(tenantId); + } + @Override public PageData findAssetsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findAssetsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 028b6ff59b6..bd8ddb40597 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -180,6 +180,11 @@ public void deleteCustomersByTenantId(TenantId tenantId) { customersByTenantRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteCustomersByTenantId(tenantId); + } + private PaginatedRemover customersByTenantRemover = new PaginatedRemover() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index e08653856b5..37abeb993cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -272,6 +272,11 @@ public void deleteDashboardsByTenantId(TenantId tenantId) { tenantDashboardsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteDashboardsByTenantId(tenantId); + } + @Override public PageData findDashboardsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDashboardsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index f9be1c9988a..ceebe8f7b25 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -359,6 +359,11 @@ public void deleteDeviceProfilesByTenantId(TenantId tenantId) { tenantDeviceProfilesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteDeviceProfilesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findDeviceProfileById(tenantId, new DeviceProfileId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 01894a8cb89..7a8daa6d7c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -422,6 +422,12 @@ public void deleteDevicesByTenantId(TenantId tenantId) { tenantDevicesRemover.removeEntities(tenantId, tenantId); } + @Transactional + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteDevicesByTenantId(tenantId); + } + @Override public PageData findDevicesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 118449b6981..0c95a6f0c7c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -266,6 +266,11 @@ public void deleteEdgesByTenantId(TenantId tenantId) { tenantEdgesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteEdgesByTenantId(tenantId); + } + @Override public PageData findEdgesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findEdgesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 6940b427944..7a5f873a25e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -344,6 +344,11 @@ public void deleteEntityViewsByTenantId(TenantId tenantId) { tenantEntityViewRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteEntityViewsByTenantId(tenantId); + } + @Override public ListenableFuture> findEntityViewTypesByTenantId(TenantId tenantId) { log.trace("Executing findEntityViewTypesByTenantId, tenantId [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 0086e179268..1097ec5ecec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; +import java.util.UUID; + @Component @RequiredArgsConstructor @Slf4j @@ -57,4 +59,11 @@ public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); } + public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { + UUID tasksKey = UUID.randomUUID(); // so that all tasks are processed synchronously from one partition + for (EntityType entityType : entityTypes) { + housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java index 85577a22964..5e55318953c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -17,8 +17,13 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import java.util.UUID; + public interface HousekeeperService { void submitTask(HousekeeperTask task); + // tasks with the same key will be pushed to the same partition and thus processed synchronously + void submitTask(UUID key, HousekeeperTask task); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java new file mode 100644 index 00000000000..6c87fa9d3ca --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + + private final EntityType entityType; + + protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { + super(tenantId, null, HousekeeperTaskType.DELETE_ENTITIES); + this.entityType = entityType; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 9fcbbfe43d8..dfcf445d8e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; import lombok.Data; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -57,4 +58,8 @@ public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId ent return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ENTITY_ALARMS); } + public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entityType) { + return new EntitiesDeletionHousekeeperTask(tenantId, entityType); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java index 22951280c18..51fb9ab9268 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; public enum HousekeeperTaskType { - DELETE_ENTITY, + DELETE_ENTITIES, DELETE_ATTRIBUTES, DELETE_TELEMETRY, // maybe divide into latest and ts kv history? DELETE_EVENTS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index 2ad0a598249..95735392970 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -104,6 +104,11 @@ public void deleteNotificationRequestsByTenantId(TenantId tenantId) { notificationRequestDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRequestsByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationRequestById(tenantId, new NotificationRequestId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java index 909febe6a8c..77a7cd96e7a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java @@ -93,6 +93,11 @@ public void deleteNotificationRulesByTenantId(TenantId tenantId) { notificationRuleDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRulesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationRuleById(tenantId, new NotificationRuleId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java index a92e4f6a4eb..5996ec6c908 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java @@ -201,6 +201,11 @@ public void deleteNotificationTargetsByTenantId(TenantId tenantId) { notificationTargetDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTargetsByTenantId(tenantId); + } + @Override public long countNotificationTargetsByTenantId(TenantId tenantId) { return notificationTargetDao.countByTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index 17b166c3ecd..6cf8239f478 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -89,6 +89,11 @@ public void deleteNotificationTemplatesByTenantId(TenantId tenantId) { notificationTemplateDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTemplatesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationTemplateById(tenantId, new NotificationTemplateId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 53a585c358f..e4a4865f026 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -230,6 +230,11 @@ public void deleteOtaPackagesByTenantId(TenantId tenantId) { tenantOtaPackageRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteOtaPackagesByTenantId(tenantId); + } + private PaginatedRemover tenantOtaPackageRemover = new PaginatedRemover<>() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index 57c45b1d0b9..ebbb69078c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -129,6 +129,11 @@ public void deleteQueuesByTenantId(TenantId tenantId) { tenantQueuesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteQueuesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findQueueById(tenantId, new QueueId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 35e13bff3a5..a80772b5069 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -203,6 +203,11 @@ public void deleteResourcesByTenantId(TenantId tenantId) { tenantResourcesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteResourcesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findResourceInfoById(tenantId, new TbResourceId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java index 54d7a2170ce..9c65d1efe9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java @@ -66,6 +66,11 @@ public void deleteAllRpcByTenantId(TenantId tenantId) { tenantRpcRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAllRpcByTenantId(tenantId); + } + @Override public Rpc findById(TenantId tenantId, RpcId rpcId) { log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, rpcId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index ebbec8969f4..fe685d61e1d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -443,6 +443,11 @@ public void deleteRuleChainsByTenantId(TenantId tenantId) { tenantRuleChainsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteRuleChainsByTenantId(tenantId); + } + @Override public RuleChainData exportTenantRuleChains(TenantId tenantId, PageLink pageLink) { Validator.validateId(tenantId, "Incorrect tenant id for search rule chain request."); diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bfc99d1b54a..1100d40d22c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -21,14 +21,20 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; +import java.util.Optional; + @Service @Slf4j -public class AdminSettingsServiceImpl implements AdminSettingsService { +public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDaoService { @Autowired private AdminSettingsDao adminSettingsDao; @@ -105,4 +111,19 @@ private void dropTokenIfProviderInfoChanged(JsonNode newJsonValue, JsonNode oldJ } } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAdminSettingsByTenantId(tenantId); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.empty(); + } + + @Override + public EntityType getEntityType() { + return EntityType.ADMIN_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index d8cf6aab329..3730718afa4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -196,8 +196,8 @@ public List> removeAll(TenantId tenantId, EntityId enti public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_type = ? and entity_id = ? " + "RETURNING attribute_type, attribute_key", entityId.getEntityType().name(), entityId.getId()).stream() - .map(deleted -> Pair.of((String) deleted.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), - (String) deleted.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) + .map(row -> Pair.of((String) row.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), + (String) row.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) .collect(Collectors.toList()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index 1feabaacb5d..914806e829e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.dao.DaoUtil; @@ -50,4 +51,9 @@ public void removeById(TenantId tenantId, UserSettingsCompositeKey id) { userSettingsRepository.deleteById(id); } + @Override + public void removeByUserId(TenantId tenantId, UserId userId) { + userSettingsRepository.deleteByUserId(userId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index cb81675a806..0a8d3e431af 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -16,9 +16,20 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; +import java.util.UUID; + public interface UserSettingsRepository extends JpaRepository { + @Transactional + @Modifying + @Query("DELETE FROM UserSettingsEntity s WHERE s.userId = :userId") + void deleteByUserId(@Param("userId") UUID userId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 6fb2a93d544..0ca5e76a93a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -211,41 +211,26 @@ public Tenant saveTenant(Tenant tenant) { return savedTenant; } - /** - * We intentionally leave this method without "Transactional" annotation due to complexity of the method. - * Ideally we should delete related entites without "paginatedRemover" logic. But in such a case we can't clear cache and send events. - * We will create separate task to make "deleteTenant" transactional. - */ + @Transactional @Override public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - entityViewService.deleteEntityViewsByTenantId(tenantId); - widgetsBundleService.deleteWidgetsBundlesByTenantId(tenantId); - widgetTypeService.deleteWidgetTypesByTenantId(tenantId); - assetService.deleteAssetsByTenantId(tenantId); - assetProfileService.deleteAssetProfilesByTenantId(tenantId); - deviceService.deleteDevicesByTenantId(tenantId); - deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); - dashboardService.deleteDashboardsByTenantId(tenantId); - customerService.deleteCustomersByTenantId(tenantId); - edgeService.deleteEdgesByTenantId(tenantId); - userService.deleteTenantAdmins(tenantId); - ruleChainService.deleteRuleChainsByTenantId(tenantId); - apiUsageStateService.deleteApiUsageStateByTenantId(tenantId); - resourceService.deleteResourcesByTenantId(tenantId); - otaPackageService.deleteOtaPackagesByTenantId(tenantId); - rpcService.deleteAllRpcByTenantId(tenantId); - queueService.deleteQueuesByTenantId(tenantId); - notificationRequestService.deleteNotificationRequestsByTenantId(tenantId); - notificationRuleService.deleteNotificationRulesByTenantId(tenantId); - notificationTemplateService.deleteNotificationTemplatesByTenantId(tenantId); - notificationTargetService.deleteNotificationTargetsByTenantId(tenantId); - adminSettingsService.deleteAdminSettingsByTenantId(tenantId); + + userService.deleteByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); + + cleanUpService.removeTenantEntities(tenantId, // don't forget to implement deleteByTenantId from EntityDaoService when adding entity type to this list + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE, EntityType.WIDGET_TYPE, + EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE, EntityType.DEVICE_PROFILE, + EntityType.DASHBOARD, EntityType.CUSTOMER, EntityType.EDGE, EntityType.RULE_CHAIN, + EntityType.API_USAGE_STATE, EntityType.TB_RESOURCE, EntityType.OTA_PACKAGE, EntityType.RPC, + EntityType.QUEUE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE, + EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.ADMIN_SETTINGS + ); + publishEvictEvent(new TenantEvictEvent(tenantId, true)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(tenantId).build()); - alarmService.deleteEntityAlarmRecordsByTenantId(tenantId); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 8d1a45789d5..1019481d678 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -172,6 +172,11 @@ public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findApiUsageStateById(tenantId, new ApiUsageStateId(entityId.getId()))); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteApiUsageStateByTenantId(tenantId); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 7f374b72591..4651aad4c0a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -85,6 +85,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic private final UserDao userDao; private final UserCredentialsDao userCredentialsDao; private final UserAuthSettingsDao userAuthSettingsDao; + private final UserSettingsDao userSettingsDao; private final DataValidator userValidator; private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; @@ -255,6 +256,7 @@ public void deleteUser(TenantId tenantId, User user) { validateId(userId, INCORRECT_USER_ID + userId); userCredentialsDao.removeByUserId(tenantId, userId); userAuthSettingsDao.removeByUserId(userId); + userSettingsDao.removeByUserId(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); @@ -313,6 +315,12 @@ public void deleteTenantAdmins(TenantId tenantId) { tenantAdminsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + log.trace("Executing deleteByTenantId, tenantId [{}]", tenantId); + usersRemover.removeEntities(tenantId, tenantId); + } + @Override public PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findCustomerUsers, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -464,6 +472,18 @@ protected void removeEntity(TenantId tenantId, User entity) { } }; + private final PaginatedRemover usersRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return findUsersByTenantId(tenantId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, User user) { + deleteUser(tenantId, user); + } + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findUserById(tenantId, new UserId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java index f2e5474c588..aac0b8f4de3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.user; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; @@ -27,4 +28,6 @@ public interface UserSettingsDao { void removeById(TenantId tenantId, UserSettingsCompositeKey key); + void removeByUserId(TenantId tenantId, UserId userId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 7f7d4d5ae8b..aad37a8f66c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -226,6 +226,11 @@ public void deleteWidgetTypesByTenantId(TenantId tenantId) { tenantWidgetTypeRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetTypesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findWidgetTypeById(tenantId, new WidgetTypeId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 9d6960a7d1e..1631bfbdbf3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -179,6 +179,11 @@ public void deleteWidgetsBundlesByTenantId(TenantId tenantId) { tenantWidgetsBundleRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetsBundlesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId()))); From 4cbd57c923f2509fe9449c1bff21c06fdf698977 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 16 Feb 2024 13:34:26 +0200 Subject: [PATCH 010/304] Telemetry deletion task; task processing timeout --- .../DefaultHousekeeperService.java | 37 ++++++++++++++----- .../HousekeeperReprocessingService.java | 31 +++++----------- .../TelemetryDeletionTaskProcessor.java | 14 ++++++- .../dao/attributes/BaseAttributesService.java | 1 + .../data/EntitiesDeletionHousekeeperTask.java | 2 +- .../dao/housekeeper/data/HousekeeperTask.java | 3 +- 6 files changed, 54 insertions(+), 34 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 9361baae2b7..10d53f35005 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -42,8 +42,12 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @TbCoreComponent @@ -57,10 +61,13 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @Value("${queue.core.housekeeper.poll-interval-ms:10000}") private int pollInterval; + private int taskProcessingTimeout = 120; private boolean stopped; @@ -89,9 +96,10 @@ public void afterStartUp() { for (TbProtoQueueMsg msg : msgs) { try { - processTask(msg.getValue()); - } catch (Exception e) { - log.error("Message processing failed", e); + processTask(msg); + } catch (Throwable e) { + log.error("Unexpected error during message processing [{}]", msg, e); + reprocessingService.submitForReprocessing(msg); } } consumer.commit(); @@ -111,18 +119,29 @@ public void afterStartUp() { } @SuppressWarnings("unchecked") - protected void processTask(ToHousekeeperServiceMsg msg) { + protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { + ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); - log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); try { - taskProcessor.process((T) task); - } catch (Exception e) { + Future future = executor.submit(() -> { + taskProcessor.process((T) task); + return null; + }); + future.get(taskProcessingTimeout, TimeUnit.SECONDS); + } catch (ExecutionException executionException) { + Throwable error = executionException.getCause(); log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - task.getTaskType(), msg.getTask().getAttempt(), task, e); - reprocessingService.submitForReprocessing(msg); + task.getTaskType(), msg.getTask().getAttempt(), task, error); + reprocessingService.submitForReprocessing(queueMsg); + } catch (TimeoutException timeoutException) { + log.error("[{}][{}][{}] {} task processing timeout after {} seconds, submitting for reprocessing (attempt {}): {}", + task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + task.getTaskType(), taskProcessingTimeout, msg.getTask().getAttempt(), task); + reprocessingService.submitForReprocessing(queueMsg); // fixme: we should schedule such task for later (separate queue?) } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index ac0051def13..9d7114476b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -32,9 +32,9 @@ import javax.annotation.PreDestroy; import java.util.List; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; @TbCoreComponent @Service @@ -46,7 +46,7 @@ public class HousekeeperReprocessingService { private final TbQueueConsumer> consumer; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing @@ -68,28 +68,17 @@ public void afterStartUp() { while (!stopped && !consumer.isStopped()) { try { List> msgs = consumer.poll(pollInterval); - if (msgs.isEmpty()) { + if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately stop(); return; } - if (msgs.stream().anyMatch(msg -> { - boolean newMsg = msg.getValue().getTask().getTs() >= startTs; - if (newMsg) { - log.info("Stopping reprocessing due to msg is new {}", msg); - } - return newMsg; - })) { - stop(); // fixme: we should commit already reprocessed messages; maybe submit for reprocessing again and commit? - // msg batch size should be 1. otherwise some tasks won't be reprocessed - return; - } for (TbProtoQueueMsg msg : msgs) { try { - housekeeperService.processTask(msg.getValue());// fixme: or should we submit to queue? - Thread.sleep(1000); + housekeeperService.processTask(msg); } catch (Exception e) { - log.error("Message processing failed", e); + log.error("Unexpected error during message reprocessing [{}]", msg, e); + submitForReprocessing(msg); } } consumer.commit(); @@ -108,20 +97,20 @@ public void afterStartUp() { log.info("Started Housekeeper tasks reprocessing"); } - public void submitForReprocessing(ToHousekeeperServiceMsg msg) { + public void submitForReprocessing(TbProtoQueueMsg queueMsg) { + ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis()) // maybe set ts + 1 hour so that no-one reprocesses it immediately + .setTs(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)) // so that it is not reprocessed by anyone in the nearest hour .build()) .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - // fixme submit with the same msg key, so that the messages goes to this consumer and will not be processed by anyone else - producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index e925c9f20c0..7159c43f46f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -16,21 +16,31 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import java.util.List; + @Component @RequiredArgsConstructor +@Slf4j public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; @Override public void process(HousekeeperTask task) throws Exception { -// timeseriesService.remove() -// timeseriesService.removeAllLatest(); + List keys = timeseriesService.findAllKeysByEntityIds(task.getTenantId(), List.of(task.getEntityId())); + for (String key : keys) { + DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); + timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); + } + log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 1cace5b71cc..9359383071d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -118,6 +118,7 @@ public ListenableFuture> removeAll(TenantId tenantId, EntityId enti @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + // todo: probably better to retrieve by scope in case there are too many attributes return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 6c87fa9d3ca..8f4a8d38b0d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -25,7 +25,7 @@ public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private final EntityType entityType; protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { - super(tenantId, null, HousekeeperTaskType.DELETE_ENTITIES); + super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); this.entityType = entityType; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index dfcf445d8e3..f9389e45667 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; import lombok.Data; +import lombok.NonNull; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; @@ -31,7 +32,7 @@ public class HousekeeperTask implements Serializable { private final HousekeeperTaskType taskType; private final long ts; - protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { + protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) { this.tenantId = tenantId; this.entityId = entityId; this.taskType = taskType; From 8e16a1562a62c87f0e321c3729c7cdd7d434c2a1 Mon Sep 17 00:00:00 2001 From: devaskim Date: Sat, 17 Feb 2024 18:58:11 +0500 Subject: [PATCH 011/304] Implemented cell click action for table widgets. --- .../widget_types/asset_admin_table.json | 2 +- .../widget_types/device_admin_table.json | 2 +- .../system/widget_types/entities_table.json | 2 +- .../entities-table-widget.component.html | 38 ++++++++++++++----- .../entity/entities-table-widget.component.ts | 30 +++++++++++++++ .../assets/locale/locale.constant-en_US.json | 2 + 6 files changed, 63 insertions(+), 13 deletions(-) diff --git a/application/src/main/data/json/system/widget_types/asset_admin_table.json b/application/src/main/data/json/system/widget_types/asset_admin_table.json index 0a2eb2c37a0..6e1dc456607 100644 --- a/application/src/main/data/json/system/widget_types/asset_admin_table.json +++ b/application/src/main/data/json/system/widget_types/asset_admin_table.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n },\n 'cellDoubleClick': {\n name: 'widget-action.cell-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-entities-table-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/device_admin_table.json b/application/src/main/data/json/system/widget_types/device_admin_table.json index af460a286c0..61aecb377d4 100644 --- a/application/src/main/data/json/system/widget_types/device_admin_table.json +++ b/application/src/main/data/json/system/widget_types/device_admin_table.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n },\n 'cellDoubleClick': {\n name: 'widget-action.cell-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-entities-table-widget-settings", diff --git a/application/src/main/data/json/system/widget_types/entities_table.json b/application/src/main/data/json/system/widget_types/entities_table.json index 1ebf8d4fde4..5b8ccc1d508 100644 --- a/application/src/main/data/json/system/widget_types/entities_table.json +++ b/application/src/main/data/json/system/widget_types/entities_table.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.entitiesTableWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n hasDataPageLink: true,\n warnOnPageDataOverflow: false,\n dataKeysOptional: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'name', type: 'entityField' }];\n }\n };\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n },\n 'rowDoubleClick': {\n name: 'widget-action.row-double-click',\n multiple: false\n },\n 'cellClick': {\n name: 'widget-action.cell-click',\n multiple: false\n },\n 'cellDoubleClick': {\n name: 'widget-action.cell-double-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-entities-table-widget-settings", diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.html index 22e79fd526c..46ea65141c9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.html @@ -42,10 +42,19 @@ matSort [matSortActive]="sortOrderProperty" [matSortDirection]="pageLinkSortDirection()" matSortDisableClear> {{ column.title }} - - + + + + + + + + - + + + + + + ): boolean { + let configured = false; + actionSourceIds.forEach(id => configured = configured || this.ctx.actionsApi.getActionDescriptors(id).length > 0 ); + return configured; + } + ngOnDestroy(): void { if (this.widgetResize$) { this.widgetResize$.disconnect(); @@ -310,6 +317,9 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni cssParser.cssPreviewNamespace = namespace; cssParser.createStyleElement(namespace, cssString); $(this.elementRef.nativeElement).addClass(namespace); + + this.useCellClickAction = this.isActionsConfigured(['cellClick', 'cellDoubleClick']) && + !this.isActionsConfigured(['rowClick', 'rowDoubleClick']); } private updateDatasources() { @@ -687,6 +697,26 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni } } + public onCellClick($event: Event, entity: EntityData, key: EntityColumn, isDouble?: boolean) { + if ($event) { + $event.stopPropagation(); + } + this.entityDatasource.toggleCurrentEntity(entity); + const actionSourceId = isDouble ? 'cellDoubleClick' : 'cellClick'; + const descriptors = this.ctx.actionsApi.getActionDescriptors(actionSourceId); + if (descriptors.length) { + let entityId; + let entityName; + let entityLabel; + if (entity) { + entityId = entity.id; + entityName = entity.entityName; + entityLabel = entity.entityLabel; + } + this.ctx.actionsApi.handleWidgetAction($event, descriptors[0], entityId, entityName, {entity, key}, entityLabel); + } + } + public onRowClick($event: Event, entity: EntityData, isDouble?: boolean) { if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 288db8c9d2e..aca909ad88d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -6911,6 +6911,7 @@ "widget-action": { "action-cell-button": "Action cell button", "row-click": "On row click", + "cell-click": "On cell click", "polygon-click": "On polygon click", "marker-click": "On marker click", "circle-click": "On circle click", @@ -6919,6 +6920,7 @@ "element-click": "On HTML element click", "pie-slice-click": "On slice click", "row-double-click": "On row double click", + "cell-double-click": "On cell double click", "card-click": "On card click", "click": "On click" } From 721854c8793e5f89c71cfdf05fb2c2320ed6a62b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 20 Feb 2024 18:50:29 +0200 Subject: [PATCH 012/304] Housekeeper: reprocessing improvements --- .../DefaultHousekeeperService.java | 20 ++--- .../HousekeeperReprocessingService.java | 52 ++++++++---- .../EventsDeletionTaskProcessor.java | 1 + common/proto/src/main/proto/queue.proto | 83 ++++++++++--------- .../provider/AwsSqsMonolithQueueFactory.java | 4 +- .../provider/AwsSqsTbCoreQueueFactory.java | 4 +- .../InMemoryMonolithQueueFactory.java | 8 +- .../provider/KafkaMonolithQueueFactory.java | 14 ++-- .../provider/KafkaTbCoreQueueFactory.java | 14 ++-- .../provider/PubSubMonolithQueueFactory.java | 4 +- .../provider/PubSubTbCoreQueueFactory.java | 4 +- .../RabbitMqMonolithQueueFactory.java | 4 +- .../provider/RabbitMqTbCoreQueueFactory.java | 4 +- .../ServiceBusMonolithQueueFactory.java | 4 +- .../ServiceBusTbCoreQueueFactory.java | 4 +- .../queue/provider/TbCoreQueueFactory.java | 5 +- .../provider/TbCoreQueueProducerProvider.java | 9 +- .../provider/TbQueueProducerProvider.java | 2 +- .../TbRuleEngineProducerProvider.java | 2 +- .../TbTransportQueueProducerProvider.java | 2 +- .../TbVersionControlProducerProvider.java | 2 +- .../queue/settings/TbQueueCoreSettings.java | 2 +- 22 files changed, 134 insertions(+), 114 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 10d53f35005..644e3cc2c7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -99,7 +99,7 @@ public void afterStartUp() { processTask(msg); } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); - reprocessingService.submitForReprocessing(msg); + reprocessingService.submitForReprocessing(msg, e); } } consumer.commit(); @@ -131,23 +131,23 @@ protected void processTask(TbProtoQueueMsg> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + private static final int startDelay = 15; // fixme - to 5 minutes + private static final int reprocessingDelay = 30; // seconds + private static final int maxReprocessingAttempts = 5; @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing private boolean stopped; // todo: stats public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, - TbCoreQueueFactory queueFactory, + PartitionService partitionService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider) { this.housekeeperService = housekeeperService; - this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.partitionService = partitionService; + this.queueFactory = queueFactory; this.producerProvider = producerProvider; } - @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) - public void afterStartUp() { + @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS) + public void startReprocessing() { + if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { + return; + } + + var consumer = queueFactory.createHousekeeperReprocessingMsgConsumer(); consumer.subscribe(); consumerExecutor.submit(() -> { - while (!stopped && !consumer.isStopped()) { + log.info("Starting Housekeeper tasks reprocessing"); + long startTs = System.currentTimeMillis(); + while (!stopped) { try { List> msgs = consumer.poll(pollInterval); if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately - stop(); - return; + break; } for (TbProtoQueueMsg msg : msgs) { @@ -78,7 +94,7 @@ public void afterStartUp() { housekeeperService.processTask(msg); } catch (Exception e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); - submitForReprocessing(msg); + submitForReprocessing(msg, e); } } consumer.commit(); @@ -93,31 +109,33 @@ public void afterStartUp() { } } } + consumer.unsubscribe(); + log.info("Stopped Housekeeper tasks reprocessing"); }); - log.info("Started Housekeeper tasks reprocessing"); } - public void submitForReprocessing(TbProtoQueueMsg queueMsg) { + // todo: dead letter queue if attempts count exceeds the configured maximum + public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)) // so that it is not reprocessed by anyone in the nearest hour + .addErrors(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)) + .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(reprocessingDelay)) .build()) .build(); - var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + log.trace("Submitting for reprocessing: {}", msg); + var producer = producerProvider.getHousekeeperReprocessingMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); } @PreDestroy private void stop() { - log.info("Stopped Housekeeper tasks reprocessing"); stopped = true; - consumer.unsubscribe(); consumerExecutor.shutdownNow(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 56edf016371..b95d7fc1e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -29,6 +29,7 @@ public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index a1be25f29d4..1a4f92fcd38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -217,12 +217,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 1ee8b404a5b..310bc75fe4e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -171,13 +171,13 @@ public TbQueueConsumer> } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { - return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { - return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 5ee9b6f53bf..b4c2220ac15 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -373,22 +373,22 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) - .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) - .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) .admin(housekeeperAdmin) .build(); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) - .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index c01ba32f705..635eb3e3006 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -330,22 +330,22 @@ public TbQueueConsumer> } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) - .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) - .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) .admin(housekeeperAdmin) .build(); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) - .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index a63a9fc16b1..e6a23b80680 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -239,12 +239,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 5f1f282e390..1b52a2ba423 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -210,12 +210,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 802b08773d4..fa89956b64a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -236,12 +236,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index ceb6e03ca5c..b1f3de672fd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -209,12 +209,12 @@ public TbQueueConsumer> } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 1e9c1cadaf2..5f4be98ca00 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -235,12 +235,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e39df45b820..579fba272c3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -210,12 +210,12 @@ public TbQueueConsumer> createHousekeep } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 279a469415c..866eb931f2e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -137,8 +136,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueConsumer> createHousekeeperMsgConsumer(); - TbQueueProducer> createHousekeeperDelayedMsgProducer(); + TbQueueProducer> createHousekeeperReprocessingMsgProducer(); - TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + TbQueueConsumer> createHousekeeperReprocessingMsgConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 4fa22cb7e39..e6bc8f04843 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -44,7 +43,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toHousekeeperDelayed; + private TbQueueProducer> toHousekeeperReprocessing; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -60,7 +59,7 @@ public void init() { this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); - this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); + this.toHousekeeperReprocessing = tbQueueProvider.createHousekeeperReprocessingMsgProducer(); } @Override @@ -104,8 +103,8 @@ public TbQueueProducer> getHousekeeperM } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { - return toHousekeeperDelayed; + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { + return toHousekeeperReprocessing; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 5cdf390f476..ff56d2e8be8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -82,6 +82,6 @@ public interface TbQueueProducerProvider { TbQueueProducer> getHousekeeperMsgProducer(); - TbQueueProducer> getHousekeeperDelayedMsgProducer(); + TbQueueProducer> getHousekeeperReprocessingMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index c5ddf57fc09..ee6700906c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -96,7 +96,7 @@ public TbQueueProducer> } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index e8dd7431a65..869c588fc74 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -92,7 +92,7 @@ public TbQueueProducer> } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index 7ddd2109ba6..3f4bd209e49 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -88,7 +88,7 @@ public TbQueueProducer> } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0b66c2fa75a..2aa01193b06 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -38,7 +38,7 @@ public class TbQueueCoreSettings { private String housekeeperTopic; @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}") - private String housekeeperDelayedTopic; + private String housekeeperReprocessingTopic; @Value("${queue.core.partitions}") private int partitions; From b72af4ead902737e490ae93cdb5696e15ca84e9a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 21 Feb 2024 18:49:15 +0200 Subject: [PATCH 013/304] Housekeeper tasks encoded to json --- .../DefaultHousekeeperService.java | 26 +++++++++---------- .../HousekeeperReprocessingService.java | 18 ++++++++++--- .../src/main/resources/thingsboard.yml | 4 +++ .../data/notification/NotificationType.java | 1 + common/proto/src/main/proto/queue.proto | 2 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 ++++ .../provider/KafkaMonolithQueueFactory.java | 6 +++-- .../provider/KafkaTbCoreQueueFactory.java | 6 +++-- .../data/AlarmsUnassignHousekeeperTask.java | 8 +++--- .../data/EntitiesDeletionHousekeeperTask.java | 8 +++--- .../dao/housekeeper/data/HousekeeperTask.java | 21 ++++++++++++--- 11 files changed, 73 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 644e3cc2c7e..76bf40fc36c 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -15,11 +15,11 @@ */ package org.thingsboard.server.service.housekeeper; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.housekeeper.HousekeeperService; @@ -40,7 +40,6 @@ import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -95,8 +94,11 @@ public void afterStartUp() { } for (TbProtoQueueMsg msg : msgs) { + log.trace("Processing task: {}", msg); try { processTask(msg); + } catch (InterruptedException e) { + return; } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); reprocessingService.submitForReprocessing(msg, e); @@ -121,16 +123,20 @@ public void afterStartUp() { @SuppressWarnings("unchecked") protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { ToHousekeeperServiceMsg msg = queueMsg.getValue(); - HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); - HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); + HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); + HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + throw new IllegalArgumentException("Unsupported task type " + task.getTaskType()); + } - log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); try { Future future = executor.submit(() -> { taskProcessor.process((T) task); return null; }); future.get(taskProcessingTimeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw e; } catch (Throwable e) { Throwable error = e; if (e instanceof ExecutionException) { @@ -152,21 +158,15 @@ public void submitTask(HousekeeperTask task) { @Override public void submitTask(UUID key, HousekeeperTask task) { + log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); producer.send(tpi, new TbProtoQueueMsg<>(key, ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() - .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .setValue(JacksonUtil.toString(task)) .setTs(task.getTs()) .setAttempt(0) .build()) .build()), null); - log.trace("[{}][{}][{}] Submitted task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); - } - - @SuppressWarnings("unchecked") - private HousekeeperTaskProcessor getTaskProcessor(HousekeeperTaskType taskType) { - return Optional.ofNullable((HousekeeperTaskProcessor) taskProcessors.get(taskType)) - .orElseThrow(() -> new IllegalArgumentException("Unsupported task type " + taskType)); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 2d2c3242f9b..44b27280891 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -28,7 +28,8 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; @@ -36,7 +37,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PreDestroy; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -90,11 +93,15 @@ public void startReprocessing() { } for (TbProtoQueueMsg msg : msgs) { + log.trace("Reprocessing task: {}", msg); try { housekeeperService.processTask(msg); - } catch (Exception e) { + } catch (InterruptedException e) { + return; + } catch (Throwable e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); submitForReprocessing(msg, e); + // fixme: msgs are duplicated } } consumer.commit(); @@ -118,12 +125,15 @@ public void startReprocessing() { public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); + int attempt = task.getAttempt() + 1; + Set errors = new LinkedHashSet<>(task.getErrorsList()); + errors.add(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)); msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .addErrors(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)) - .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(reprocessingDelay)) + .clearErrors().addAllErrors(errors) + .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis((long) (reprocessingDelay * 0.8))) .build()) .build(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7af241d41ef..f59e3c86b71 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1403,6 +1403,9 @@ queue: tb_housekeeper: - key: max.poll.records value: "1" + tb_housekeeper.reprocessing: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1425,6 +1428,7 @@ queue: # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" + housekeeper-reprocessing: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java index cfc91d8fca3..5a8fd87933f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java @@ -31,4 +31,5 @@ public enum NotificationType { RATE_LIMITS, EDGE_CONNECTION, EDGE_COMMUNICATION_FAILURE + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 73cea54a473..8e757a3b14d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1424,7 +1424,7 @@ message ToHousekeeperServiceMsg { } message HousekeeperTaskProto { - bytes value = 1; + string value = 1; int64 ts = 2; // reprocessing diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d3a5eb821d7..915f61c8b46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -45,6 +45,8 @@ public class TbKafkaTopicConfigs { private String vcProperties; @Value("${queue.kafka.topic-properties.housekeeper:}") private String housekeeperProperties; + @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") + private String housekeeperReprocessingProperties; @Getter private Map coreConfigs; @@ -66,6 +68,8 @@ public class TbKafkaTopicConfigs { private Map vcConfigs; @Getter private Map housekeeperConfigs; + @Getter + private Map housekeeperReprocessingConfigs; @PostConstruct private void init() { @@ -81,6 +85,7 @@ private void init() { fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); + housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index c8a11e1f321..95bf2d11953 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -84,6 +84,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -118,6 +119,7 @@ public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafk this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -378,7 +380,7 @@ public TbQueueProducer> createHousekeep .settings(kafkaSettings) .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .build(); } @@ -390,7 +392,7 @@ public TbQueueConsumer> createHousekeep .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .statsService(consumerStatsService) .build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index f2812564e5d..478963d3001 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -82,6 +82,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -115,6 +116,7 @@ public KafkaTbCoreQueueFactory(TopicService topicService, this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -335,7 +337,7 @@ public TbQueueProducer> .settings(kafkaSettings) .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .build(); } @@ -347,7 +349,7 @@ public TbQueueConsumer> .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .statsService(consumerStatsService) .build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index aa1858f59fc..aea09c093c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -15,13 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.User; -@Getter +@Data +@EqualsAndHashCode(callSuper = true) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { - private final String userTitle; + private String userTitle; protected AlarmsUnassignHousekeeperTask(User user) { super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 8f4a8d38b0d..7846feb2298 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -15,14 +15,16 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; -@Getter +@Data +@EqualsAndHashCode(callSuper = true) public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { - private final EntityType entityType; + private EntityType entityType; protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index f9389e45667..6b63d43d435 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,7 +15,13 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AccessLevel; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.NonNull; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -24,13 +30,20 @@ import java.io.Serializable; +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "taskType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = HousekeeperTask.class) +@JsonSubTypes({ + @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), + @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) +}) @Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class HousekeeperTask implements Serializable { - private final TenantId tenantId; - private final EntityId entityId; - private final HousekeeperTaskType taskType; - private final long ts; + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private long ts; protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) { this.tenantId = tenantId; From 7cc599b5b59ab0f65b78049e7c5b2db729618c9a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 13:58:15 +0200 Subject: [PATCH 014/304] Housekeeper stats; Grafana dashboard --- .../DefaultHousekeeperService.java | 23 +- .../HousekeeperReprocessingService.java | 33 +- .../AlarmsUnassignTaskProcessor.java | 2 +- .../AttributesDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 3 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../stats/HousekeeperStatsService.java | 117 ++++++ .../src/main/resources/thingsboard.yml | 11 +- .../common/stats/DefaultStatsFactory.java | 20 +- .../server/common/stats/StatsFactory.java | 5 + .../server/common/stats/StatsType.java | 7 +- .../provisioning/dashboards/housekeeper.json | 390 ++++++++++++++++++ 12 files changed, 588 insertions(+), 27 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java create mode 100644 docker/monitoring/grafana/provisioning/dashboards/housekeeper.json diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 76bf40fc36c..a324f279fbe 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -33,9 +33,9 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; +import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import javax.annotation.PreDestroy; import java.util.List; @@ -59,27 +59,28 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueConsumer> consumer; private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; - private final DataDecodingEncodingService dataDecodingEncodingService; + private final HousekeeperStatsService statsService; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); - @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") + private int taskProcessingTimeout; + @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - private int taskProcessingTimeout = 120; private boolean stopped; public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, - DataDecodingEncodingService dataDecodingEncodingService, + HousekeeperStatsService statsService, @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; + this.statsService = statsService; this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); - this.dataDecodingEncodingService = dataDecodingEncodingService; } @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @@ -129,18 +130,23 @@ protected void processTask(TbProtoQueueMsg future = executor.submit(() -> { taskProcessor.process((T) task); return null; }); - future.get(taskProcessingTimeout, TimeUnit.SECONDS); + future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); + statsService.reportProcessed(task, msg); } catch (InterruptedException e) { throw e; } catch (Throwable e) { Throwable error = e; if (e instanceof ExecutionException) { - error = error.getCause(); + error = e.getCause(); } else if (e instanceof TimeoutException) { error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds"); } @@ -148,6 +154,7 @@ protected void processTask(TbProtoQueueMsg { + try { + startReprocessing(); + } catch (Throwable e) { + log.error("Unexpected error during reprocessing", e); + } + }, startDelay, reprocessingDelay, TimeUnit.SECONDS); + } + public void startReprocessing() { if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { return; @@ -146,6 +158,7 @@ public void submitForReprocessing(TbProtoQueueMsg queue @PreDestroy private void stop() { stopped = true; + scheduler.shutdownNow(); consumerExecutor.shutdownNow(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 247466af65e..68fd05deec3 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -36,7 +36,7 @@ public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); - log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); + log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index e3bac35e9b5..0467e50aeb3 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -32,7 +32,7 @@ public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor @Override public void process(HousekeeperTask task) throws Exception { int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); - log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index b95d7fc1e3c..481f9f1bbf0 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -24,16 +24,17 @@ @Component @RequiredArgsConstructor public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; @Override public void process(HousekeeperTask task) throws Exception { eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); - throw new RuntimeException("test error"); } @Override public HousekeeperTaskType getTaskType() { return HousekeeperTaskType.DELETE_EVENTS; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index 7159c43f46f..e7da7402610 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -40,7 +40,7 @@ public void process(HousekeeperTask task) throws Exception { DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); } - log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); + log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java new file mode 100644 index 00000000000..4fcc13837dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.stats; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class HousekeeperStatsService { + + private final Map stats = new EnumMap<>(HousekeeperTaskType.class); + + public HousekeeperStatsService(StatsFactory statsFactory) { + for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) { + stats.put(taskType, new HousekeeperStats(taskType, statsFactory)); + } + } + + @Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS) + private void reportStats() { + String statsStr = stats.values().stream().map(stats -> { + String countersStr = stats.getCounters().stream() + .filter(counter -> counter.get() > 0) + .map(counter -> counter.getName() + " = " + counter.get()) + .collect(Collectors.joining(", ")); + if (countersStr.isEmpty()) { + return null; + } else { + return stats.getTaskType() + " {" + countersStr + "}"; + } + }).filter(Objects::nonNull).collect(Collectors.joining("; ")); + + if (!statsStr.isEmpty()) { + stats.values().forEach(HousekeeperStats::reset); + log.info("Housekeeper stats: {}", statsStr); + } + } + + public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getProcessedCounter().increment(); + } else { + stats.getReprocessedCounter().increment(); + } + } + + public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getFailedProcessingCounter().increment(); + } else { + stats.getFailedReprocessingCounter().increment(); + } + } + + @Getter + static class HousekeeperStats { + private final HousekeeperTaskType taskType; + private final List counters = new ArrayList<>(); + + private final StatsCounter processedCounter; + private final StatsCounter failedProcessingCounter; + private final StatsCounter reprocessedCounter; + private final StatsCounter failedReprocessingCounter; + + public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) { + this.taskType = taskType; + this.processedCounter = register("processed", statsFactory); + this.failedProcessingCounter = register("failedProcessing", statsFactory); + this.reprocessedCounter = register("reprocessed", statsFactory); + this.failedReprocessingCounter = register("failedReprocessing", statsFactory); + } + + private StatsCounter register(String statsName, StatsFactory statsFactory) { + StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name())); + counters.add(counter); + return counter; + } + + public void reset() { + counters.forEach(DefaultCounter::clear); + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f59e3c86b71..4ced0655f0f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1585,9 +1585,14 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: - topic: "tb_housekeeper" - reprocessing-topic: "tb_housekeeper.reprocessing" - poll-interval-ms: "1000" + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes + task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}" + vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index a6ec01d0897..459f2e0a686 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -25,6 +25,10 @@ import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -62,10 +66,24 @@ public void init() { @Override public StatsCounter createStatsCounter(String key, String statsName) { + return createStatsCounter(key, statsName, Collections.emptyMap()); + } + + @Override + public StatsCounter createStatsCounter(String key, String statsName, Map tags) { + String[] tagsArr = new String[]{STATS_NAME_TAG, statsName}; + if (!tags.isEmpty()) { + List tagsList = new ArrayList<>(List.of(tagsArr)); + tags.forEach((name, value) -> { + tagsList.add(name); + tagsList.add(value); + }); + tagsArr = tagsList.toArray(String[]::new); + } return new StatsCounter( new AtomicInteger(0), metricsEnabled ? - meterRegistry.counter(key, STATS_NAME_TAG, statsName) + meterRegistry.counter(key, tagsArr) : STUB_COUNTER, statsName ); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index 8e6989016ba..90a4191ee2a 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -17,9 +17,14 @@ import io.micrometer.core.instrument.Timer; +import java.util.Map; + public interface StatsFactory { + StatsCounter createStatsCounter(String key, String statsName); + StatsCounter createStatsCounter(String key, String statsName, Map tags); + DefaultCounter createDefaultCounter(String key, String... tags); T createGauge(String key, T number, String... tags); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 318f0418def..6c854ac8d6b 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -16,7 +16,12 @@ package org.thingsboard.server.common.stats; public enum StatsType { - RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"); + RULE_ENGINE("ruleEngine"), + CORE("core"), + TRANSPORT("transport"), + JS_INVOKE("jsInvoke"), + RATE_EXECUTOR("rateExecutor"), + HOUSEKEEPER("housekeeper"); private String name; diff --git a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json new file mode 100644 index 00000000000..8c68cc9428f --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json @@ -0,0 +1,390 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processing failures", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessing failures", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Housekeeper", + "uid": "JFJb7voIz", + "version": 5, + "weekStart": "" +} \ No newline at end of file From 85f54200195d4b8e8dca78aecb4b77492c15aa14 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 14:03:16 +0200 Subject: [PATCH 015/304] Revert proto formatting --- common/proto/src/main/proto/queue.proto | 78 ++++++++++++------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8e757a3b14d..3ac8c90a1d2 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -24,36 +24,36 @@ option java_outer_classname = "TransportProtos"; * Common data structures */ enum EntityTypeProto { - UNSPECIFIED = 0; - TENANT = 1; - CUSTOMER = 2; - USER = 3; - DASHBOARD = 4; - ASSET = 5; - DEVICE = 6; - ALARM = 7; + UNSPECIFIED = 0; + TENANT = 1; + CUSTOMER = 2; + USER = 3; + DASHBOARD = 4; + ASSET = 5; + DEVICE = 6; + ALARM = 7; // next 3 reserved for PE; - RULE_CHAIN = 11; - RULE_NODE = 12; + RULE_CHAIN = 11; + RULE_NODE = 12; // next 2 reserved for PE; - ENTITY_VIEW = 15; - WIDGETS_BUNDLE = 16; - WIDGET_TYPE = 17; + ENTITY_VIEW = 15; + WIDGETS_BUNDLE = 16; + WIDGET_TYPE = 17; // next 2 reserved for PE; - TENANT_PROFILE = 20; - DEVICE_PROFILE = 21; - ASSET_PROFILE = 22; - API_USAGE_STATE = 23; - TB_RESOURCE = 24; - OTA_PACKAGE = 25; - EDGE = 26; - RPC = 27; - QUEUE = 28; - NOTIFICATION_TARGET = 29; + TENANT_PROFILE = 20; + DEVICE_PROFILE = 21; + ASSET_PROFILE = 22; + API_USAGE_STATE = 23; + TB_RESOURCE = 24; + OTA_PACKAGE = 25; + EDGE = 26; + RPC = 27; + QUEUE = 28; + NOTIFICATION_TARGET = 29; NOTIFICATION_TEMPLATE = 30; - NOTIFICATION_REQUEST = 31; - NOTIFICATION = 32; - NOTIFICATION_RULE = 33; + NOTIFICATION_REQUEST = 31; + NOTIFICATION = 32; + NOTIFICATION_RULE = 33; } /** @@ -133,9 +133,9 @@ message KeyValueProto { } enum AttributeScopeProto { - CLIENT_SCOPE = 0; - SERVER_SCOPE = 1; - SHARED_SCOPE = 2; + CLIENT_SCOPE = 0; + SERVER_SCOPE = 1; + SHARED_SCOPE = 2; } message AttributeKey { @@ -259,7 +259,7 @@ message GetOrCreateDeviceFromGatewayResponseMsg { } enum TransportApiRequestErrorCode { - UNKNOWN_TRANSPORT_API_ERROR = 0; + UNKNOWN_TRANSPORT_API_ERROR = 0; ENTITY_LIMIT = 1; } @@ -918,15 +918,15 @@ message FromDeviceRPCResponseProto { } enum ComponentLifecycleEvent { - CREATED = 0; - STARTED = 1; - ACTIVATED = 2; - SUSPENDED = 3; - UPDATED = 4; - STOPPED = 5; - DELETED = 6; - FAILED = 7; - DEACTIVATED = 8; + CREATED = 0; + STARTED = 1; + ACTIVATED = 2; + SUSPENDED = 3; + UPDATED = 4; + STOPPED = 5; + DELETED = 6; + FAILED = 7; + DEACTIVATED = 8; } message ComponentLifecycleMsgProto { From 886c448d61afc621da9b91c32139cf14c3e5725f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 14:57:40 +0200 Subject: [PATCH 016/304] Clean up tenant service --- .../DefaultHousekeeperService.java | 18 +-- .../HousekeeperReprocessingService.java | 10 +- .../stats/HousekeeperStatsService.java | 8 +- ...mponentLifecycleEventTriggerProcessor.java | 2 +- .../server/common/data/EntityType.java | 3 +- .../dao/device/DeviceProfileServiceImpl.java | 6 - .../dao/housekeeper/CleanUpService.java | 8 +- .../data/AlarmsUnassignHousekeeperTask.java | 3 + .../data/EntitiesDeletionHousekeeperTask.java | 3 + .../validator/AssetProfileDataValidator.java | 1 + .../validator/DashboardDataValidator.java | 2 + .../validator/DeviceDataValidator.java | 2 + .../validator/DeviceProfileDataValidator.java | 1 + .../validator/ResourceDataValidator.java | 1 + .../validator/RuleChainDataValidator.java | 1 + .../settings/AdminSettingsServiceImpl.java | 39 ++---- .../server/dao/tenant/TenantServiceImpl.java | 113 +++--------------- .../usagerecord/ApiUsageStateServiceImpl.java | 2 +- 18 files changed, 63 insertions(+), 160 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index a324f279fbe..c9054c30fad 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -97,12 +97,12 @@ public void afterStartUp() { for (TbProtoQueueMsg msg : msgs) { log.trace("Processing task: {}", msg); try { - processTask(msg); + processTask(msg.getValue()); } catch (InterruptedException e) { return; } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); - reprocessingService.submitForReprocessing(msg, e); + reprocessingService.submitForReprocessing(msg.getValue(), e); } } consumer.commit(); @@ -122,8 +122,7 @@ public void afterStartUp() { } @SuppressWarnings("unchecked") - protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { - ToHousekeeperServiceMsg msg = queueMsg.getValue(); + protected void processTask(ToHousekeeperServiceMsg msg) throws Exception { HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(task.getTaskType()); if (taskProcessor == null) { @@ -131,8 +130,9 @@ protected void processTask(TbProtoQueueMsg future = executor.submit(() -> { @@ -140,7 +140,7 @@ protected void processTask(TbProtoQueueMsg void processTask(TbProtoQueueMsg msg : msgs) { log.trace("Reprocessing task: {}", msg); try { - housekeeperService.processTask(msg); + housekeeperService.processTask(msg.getValue()); } catch (InterruptedException e) { return; } catch (Throwable e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); - submitForReprocessing(msg, e); + submitForReprocessing(msg.getValue(), e); // fixme: msgs are duplicated } } @@ -134,8 +135,7 @@ public void startReprocessing() { } // todo: dead letter queue if attempts count exceeds the configured maximum - public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { - ToHousekeeperServiceMsg msg = queueMsg.getValue(); + public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; @@ -152,7 +152,7 @@ public void submitForReprocessing(TbProtoQueueMsg queue log.trace("Submitting for reprocessing: {}", msg); var producer = producerProvider.getHousekeeperReprocessingMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); // reprocessing topic has single partition, so we don't care about the msg key } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 4fcc13837dc..ab4f699ed14 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -67,8 +67,8 @@ private void reportStats() { } } - public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { - HousekeeperStats stats = this.stats.get(task.getTaskType()); + public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getProcessedCounter().increment(); } else { @@ -76,8 +76,8 @@ public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { } } - public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) { - HousekeeperStats stats = this.stats.get(task.getTaskType()); + public void reportFailure(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getFailedProcessingCounter().increment(); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java index 4c92e9b92b4..fc573e0bc4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java @@ -47,7 +47,7 @@ public boolean matchesFilter(RuleEngineComponentLifecycleEventTrigger trigger, R return false; } } - if (!partitionService.resolve(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId()).isMyPartition()) { + if (!partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId())) { return false; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 6316ab12819..fb4fe1011e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -58,8 +58,7 @@ public String getNormalName () { NOTIFICATION_TEMPLATE (30), NOTIFICATION_REQUEST (31), NOTIFICATION (32), - NOTIFICATION_RULE (33), - ADMIN_SETTINGS(34); + NOTIFICATION_RULE (33); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index ceebe8f7b25..c9806a03c4e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,7 +18,6 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -47,7 +46,6 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -91,10 +89,6 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; - @Lazy - @Autowired - private QueueService queueService; - @Autowired private ImageService imageService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 1097ec5ecec..5bdcd341b86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -37,13 +37,11 @@ public class CleanUpService { private final HousekeeperService housekeeperService; private final RelationService relationService; - @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService + @TransactionalEventListener(fallbackExecution = true) public void handleEntityDeletionEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); - log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); - - log.info("[{}][{}][{}] Handling DeleteEntityEvent", tenantId, entityId.getEntityType(), entityId.getId()); + log.trace("[{}] Handling entity deletion event: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); @@ -60,7 +58,7 @@ public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { - UUID tasksKey = UUID.randomUUID(); // so that all tasks are processed synchronously from one partition + UUID tasksKey = UUID.randomUUID(); // so that all tasks are pushed to single partition to be processed synchronously for (EntityType entityType : entityTypes) { housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index aea09c093c0..8761229e89a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.User; @Data @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { private String userTitle; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 7846feb2298..e20f3ffe0c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -15,13 +15,16 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @Data @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private EntityType entityType; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java index 19a04fd8679..7543c247e0f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java @@ -42,6 +42,7 @@ public class AssetProfileDataValidator extends DataValidator { @Lazy private AssetProfileService assetProfileService; @Autowired + @Lazy private TenantService tenantService; @Lazy @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java index 178905f7f64..829da4d5c7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; @@ -28,6 +29,7 @@ public class DashboardDataValidator extends DataValidator { @Autowired + @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index e2047dc4488..a8f1bdd9721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -39,6 +40,7 @@ public class DeviceDataValidator extends AbstractHasOtaPackageValidator private DeviceDao deviceDao; @Autowired + @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 23f3a6bbdb4..a06fd5edaae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -86,6 +86,7 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { private WidgetTypeDao widgetTypeDao; @Autowired + @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java index 222aaa3eeb7..2f758b1b9c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java @@ -53,6 +53,7 @@ public class RuleChainDataValidator extends DataValidator { private RuleChainService ruleChainService; @Autowired + @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 1100d40d22c..095e51082a8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -21,21 +21,15 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AdminSettingsId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; -import java.util.Optional; - @Service @Slf4j -public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDaoService { - +public class AdminSettingsServiceImpl implements AdminSettingsService { + @Autowired private AdminSettingsDao adminSettingsDao; @@ -46,7 +40,7 @@ public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDao public AdminSettings findAdminSettingsById(TenantId tenantId, AdminSettingsId adminSettingsId) { log.trace("Executing findAdminSettingsById [{}]", adminSettingsId); Validator.validateId(adminSettingsId, "Incorrect adminSettingsId " + adminSettingsId); - return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); + return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); } @Override @@ -65,15 +59,15 @@ public AdminSettings findAdminSettingsByTenantIdAndKey(TenantId tenantId, String public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if (adminSettings.getKey().equals("mail")){ + if (adminSettings.getKey().equals("mail")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { JsonNode newJsonValue = adminSettings.getJsonValue(); JsonNode oldJsonValue = mailSettings.getJsonValue(); - if (!newJsonValue.has("password") && oldJsonValue.has("password")){ - ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); + if (!newJsonValue.has("password") && oldJsonValue.has("password")) { + ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); } - if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")){ + if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")) { ((ObjectNode) newJsonValue).put("refreshToken", oldJsonValue.get("refreshToken").asText()); } dropTokenIfProviderInfoChanged(newJsonValue, oldJsonValue); @@ -98,12 +92,12 @@ public void deleteAdminSettingsByTenantId(TenantId tenantId) { } private void dropTokenIfProviderInfoChanged(JsonNode newJsonValue, JsonNode oldJsonValue) { - if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()){ + if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()) { if (!newJsonValue.get("providerId").equals(oldJsonValue.get("providerId")) || !newJsonValue.get("clientId").equals(oldJsonValue.get("clientId")) || !newJsonValue.get("clientSecret").equals(oldJsonValue.get("clientSecret")) || !newJsonValue.get("redirectUri").equals(oldJsonValue.get("redirectUri")) || - (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))){ + (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))) { ((ObjectNode) newJsonValue).put("tokenGenerated", false); ((ObjectNode) newJsonValue).remove("refreshToken"); ((ObjectNode) newJsonValue).remove("refreshTokenExpires"); @@ -111,19 +105,4 @@ private void dropTokenIfProviderInfoChanged(JsonNode newJsonValue, JsonNode oldJ } } - @Override - public void deleteByTenantId(TenantId tenantId) { - deleteAdminSettingsByTenantId(tenantId); - } - - @Override - public Optional> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.empty(); - } - - @Override - public EntityType getEntityType() { - return EntityType.ADMIN_SETTINGS; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index b82cfa8613f..6ab9361b192 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -16,9 +16,8 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -34,32 +33,17 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.customer.CustomerService; -import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; -import org.thingsboard.server.dao.notification.NotificationRequestService; -import org.thingsboard.server.dao.notification.NotificationRuleService; import org.thingsboard.server.dao.notification.NotificationSettingsService; -import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.notification.NotificationTemplateService; -import org.thingsboard.server.dao.ota.OtaPackageService; -import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.resource.ResourceService; -import org.thingsboard.server.dao.rpc.RpcService; -import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.dao.widget.WidgetTypeService; -import org.thingsboard.server.dao.widget.WidgetsBundleService; import java.util.List; import java.util.Optional; @@ -68,89 +52,22 @@ @Service("TenantDaoService") @Slf4j +@RequiredArgsConstructor public class TenantServiceImpl extends AbstractCachedEntityService implements TenantService { private static final String DEFAULT_TENANT_REGION = "Global"; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - @Autowired - private TenantDao tenantDao; - - @Autowired - private TenantProfileService tenantProfileService; - - @Autowired - @Lazy - private UserService userService; - - @Autowired - private CustomerService customerService; - - @Autowired - private AssetService assetService; - - @Autowired - private AssetProfileService assetProfileService; - - @Autowired - private DeviceService deviceService; - - @Autowired - private DeviceProfileService deviceProfileService; - - @Lazy - @Autowired - private ApiUsageStateService apiUsageStateService; - - @Autowired - private WidgetsBundleService widgetsBundleService; - - @Autowired - private WidgetTypeService widgetTypeService; - - @Autowired - private DashboardService dashboardService; - - @Autowired - private RuleChainService ruleChainService; - - @Autowired - private ResourceService resourceService; - - @Autowired - @Lazy - private OtaPackageService otaPackageService; - - @Autowired - private RpcService rpcService; - - @Autowired - private DataValidator tenantValidator; - - @Lazy - @Autowired - private QueueService queueService; - - @Autowired - private AdminSettingsService adminSettingsService; - - @Autowired - private NotificationSettingsService notificationSettingsService; - - @Autowired - private NotificationRequestService notificationRequestService; - - @Autowired - private NotificationRuleService notificationRuleService; - - @Autowired - private NotificationTemplateService notificationTemplateService; - - @Autowired - private NotificationTargetService notificationTargetService; - - @Autowired - protected TbTransactionalCache existsTenantCache; + private final TenantDao tenantDao; + private final TenantProfileService tenantProfileService; + private final UserService userService; + private final AssetProfileService assetProfileService; + private final DeviceProfileService deviceProfileService; + private final ApiUsageStateService apiUsageStateService; + private final AdminSettingsService adminSettingsService; + private final NotificationSettingsService notificationSettingsService; + private final DataValidator tenantValidator; + private final TbTransactionalCache existsTenantCache; @TransactionalEventListener(classes = TenantEvictEvent.class) @Override @@ -218,16 +135,18 @@ public void deleteTenant(TenantId tenantId) { Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); userService.deleteByTenantId(tenantId); + adminSettingsService.deleteAdminSettingsByTenantId(tenantId); + notificationSettingsService.deleteNotificationSettings(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); - notificationSettingsService.deleteNotificationSettings(tenantId); // fixme: maybe remove ADMIN_SETTINGS entity type, just delete here or in the CleanUpService + // fixme: we should publish to housekeeper when a transaction is committed (or better after msg was broadcasted): move this to cleanup-service cleanUpService.removeTenantEntities(tenantId, // don't forget to implement deleteByTenantId from EntityDaoService when adding entity type to this list EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE, EntityType.WIDGET_TYPE, EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.DASHBOARD, EntityType.CUSTOMER, EntityType.EDGE, EntityType.RULE_CHAIN, EntityType.API_USAGE_STATE, EntityType.TB_RESOURCE, EntityType.OTA_PACKAGE, EntityType.RPC, EntityType.QUEUE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE, - EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.ADMIN_SETTINGS + EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET ); publishEvictEvent(new TenantEvictEvent(tenantId, true)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 1019481d678..0b880b7c5fb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -59,7 +59,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final DataValidator apiUsageStateValidator; public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, - TenantService tenantService, @Lazy TimeseriesService tsService, + @Lazy TenantService tenantService, @Lazy TimeseriesService tsService, DataValidator apiUsageStateValidator) { this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; From b20b6006e9b79e06267e077e1c36cb58e2e9c00a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 1 Mar 2024 14:14:25 +0200 Subject: [PATCH 017/304] Notifications to sysadmin on housekeeper task processing failure --- .../install/ThingsboardInstallService.java | 7 +- .../DefaultHousekeeperService.java | 19 ++++-- .../AlarmsUnassignTaskProcessor.java | 4 +- .../AttributesDeletionTaskProcessor.java | 4 +- .../EntitiesDeletionTaskProcessor.java | 4 +- .../EntityAlarmsDeletionTaskProcessor.java | 4 +- .../EventsDeletionTaskProcessor.java | 4 +- .../processor/HousekeeperTaskProcessor.java | 4 +- .../TelemetryDeletionTaskProcessor.java | 4 +- .../stats/HousekeeperStatsService.java | 3 +- .../DefaultSystemDataLoaderService.java | 37 ++++++----- .../install/SystemDataLoaderService.java | 2 +- ...TaskProcessingFailureTriggerProcessor.java | 53 +++++++++++++++ .../NotificationTemplateService.java | 2 + .../AlarmsUnassignHousekeeperTask.java | 2 +- .../EntitiesDeletionHousekeeperTask.java | 9 ++- .../data/housekeeper}/HousekeeperTask.java | 8 ++- .../data/housekeeper/HousekeeperTaskType.java | 34 ++++++++++ .../data/notification/NotificationType.java | 3 +- ...TaskProcessingFailureNotificationInfo.java | 61 +++++++++++++++++ .../trigger/TaskProcessingFailureTrigger.java | 65 +++++++++++++++++++ .../config/NotificationRuleTriggerConfig.java | 1 + .../config/NotificationRuleTriggerType.java | 3 +- ...gFailureNotificationRuleTriggerConfig.java | 21 +++--- .../dao/housekeeper/CleanUpService.java | 28 +++++--- .../dao/housekeeper/HousekeeperService.java | 6 +- .../DefaultNotificationSettingsService.java | 24 +++---- .../DefaultNotificationTemplateService.java | 5 ++ .../notification/DefaultNotifications.java | 14 ++++ .../notification/NotificationTemplateDao.java | 2 + .../JpaNotificationTemplateDao.java | 5 ++ .../NotificationTemplateRepository.java | 5 ++ .../rule-notification-dialog.component.html | 13 ++++ .../rule-notification-dialog.component.ts | 13 +++- .../template-notification-dialog.component.ts | 3 +- .../app/shared/models/notification.models.ts | 18 +++-- .../notification/task_processing_failure.md | 46 +++++++++++++ .../assets/locale/locale.constant-en_US.json | 5 +- 38 files changed, 458 insertions(+), 87 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/AlarmsUnassignHousekeeperTask.java (95%) rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/EntitiesDeletionHousekeeperTask.java (83%) rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/HousekeeperTask.java (91%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java rename dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java => common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java (61%) create mode 100644 ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 44d81902a84..a79ed50eb74 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -110,7 +110,7 @@ public void performInstall() { log.info("Upgrading ThingsBoard from version 3.5.1 to 3.6.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.5.1"); dataUpdateService.updateData("3.5.1"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); @@ -126,7 +126,10 @@ public void performInstall() { case "3.6.2": log.info("Upgrading ThingsBoard from version 3.6.2 to 3.6.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); + case "3.6.3": + log.info("Upgrading ThingsBoard from version 3.6.3 to 3.6.4 ..."); + systemDataLoaderService.updateDefaultNotificationConfigs(false); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index c9054c30fad..56123af64dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -21,10 +21,12 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.housekeeper.HousekeeperService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -60,6 +62,7 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; private final HousekeeperStatsService statsService; + private final NotificationRuleProcessor notificationRuleProcessor; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @@ -75,11 +78,13 @@ public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingServ TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, HousekeeperStatsService statsService, + NotificationRuleProcessor notificationRuleProcessor, @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; this.statsService = statsService; + this.notificationRuleProcessor = notificationRuleProcessor; this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); } @@ -155,14 +160,14 @@ protected void processTask(ToHousekeeperServiceMsg m task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(msg, error); statsService.reportFailure(task.getTaskType(), msg); + notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() + .task(task) + .error(error) + .attempt(msg.getTask().getAttempt()) + .build()); } } - @Override - public void submitTask(HousekeeperTask task) { - submitTask(task.getEntityId().getId(), task); - } - @Override public void submitTask(UUID key, HousekeeperTask task) { log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 68fd05deec3..9a8517903fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -20,8 +20,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; -import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 0467e50aeb3..c8ff5306a7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -19,8 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index da7af3c42be..8028a761997 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -19,8 +19,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; -import org.thingsboard.server.dao.housekeeper.data.EntitiesDeletionHousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java index a91f2262f98..974a95eb085 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -18,8 +18,8 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 481f9f1bbf0..551c8ce7e04 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -18,8 +18,8 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index 7daa99540e3..79e87fcc91d 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.housekeeper.processor; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; public interface HousekeeperTaskProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index e7da7402610..c7f37ed0910 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -20,8 +20,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index ab4f699ed14..3f6e88a2994 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -23,8 +23,7 @@ import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import java.util.ArrayList; diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index b20a548d900..45ae1280b89 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -548,7 +548,7 @@ private void save(DeviceId deviceId, String key, boolean value) { } else { ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) - , System.currentTimeMillis()))); + , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -692,23 +692,26 @@ public void createDefaultNotificationConfigs() { @Override @SneakyThrows - public void updateDefaultNotificationConfigs() { - PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); - ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); - log.info("Updating default edge failure notification configs for all tenants"); - AtomicInteger count = new AtomicInteger(); - for (TenantId tenantId : tenants) { - executor.submit(() -> { - notificationSettingsService.updateDefaultNotificationConfigs(tenantId); - int n = count.incrementAndGet(); - if (n % 500 == 0) { - log.info("{} tenants processed", n); - } - }); - } - executor.shutdown(); - executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + public void updateDefaultNotificationConfigs(boolean updateTenants) { + log.info("Updating notification configs..."); notificationSettingsService.updateDefaultNotificationConfigs(TenantId.SYS_TENANT_ID); + + if (updateTenants) { + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); + AtomicInteger count = new AtomicInteger(); + for (TenantId tenantId : tenants) { + executor.submit(() -> { + notificationSettingsService.updateDefaultNotificationConfigs(tenantId); + int n = count.incrementAndGet(); + if (n % 500 == 0) { + log.info("{} tenants processed", n); + } + }); + } + executor.shutdown(); + executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index eeac1b6aa46..7c05ff620fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -35,6 +35,6 @@ public interface SystemDataLoaderService { void createDefaultNotificationConfigs(); - void updateDefaultNotificationConfigs(); + void updateDefaultNotificationConfigs(boolean updateTenants); } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java new file mode 100644 index 00000000000..77546902159 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.notification.rule.trigger; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.notification.info.TaskProcessingFailureNotificationInfo; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; + +@Service +public class TaskProcessingFailureTriggerProcessor implements NotificationRuleTriggerProcessor { + + @Override + public boolean matchesFilter(TaskProcessingFailureTrigger trigger, TaskProcessingFailureNotificationRuleTriggerConfig triggerConfig) { + return true; + } + + @Override + public TaskProcessingFailureNotificationInfo constructNotificationInfo(TaskProcessingFailureTrigger trigger) { + HousekeeperTask task = trigger.getTask(); + return TaskProcessingFailureNotificationInfo.builder() + .tenantId(task.getTenantId()) + .entityId(task.getEntityId()) + .taskType(task.getTaskType()) + .taskDescription(task.getDescription()) + .error(StringUtils.truncate(ExceptionUtils.getStackTrace(trigger.getError()), 1024)) + .attempt(trigger.getAttempt()) + .build(); + } + + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java index 1f4019063b9..664f9769043 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java @@ -32,6 +32,8 @@ public interface NotificationTemplateService { PageData findNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id); void deleteNotificationTemplatesByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 8761229e89a..af4e7469872 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; import lombok.AccessLevel; import lombok.Data; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java similarity index 83% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index e20f3ffe0c2..515703991c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; @@ -34,4 +35,10 @@ protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityTy this.entityType = entityType; } + @JsonIgnore + @Override + public String getDescription() { + return entityType.getNormalName().toLowerCase() + "s deletion"; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 6b63d43d435..44598d511ce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -76,4 +77,9 @@ public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entit return new EntitiesDeletionHousekeeperTask(tenantId, entityType); } + @JsonIgnore + public String getDescription() { + return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java new file mode 100644 index 00000000000..4f30f94a456 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Getter +public enum HousekeeperTaskType { + + DELETE_ENTITIES("entities deletion"), + DELETE_ATTRIBUTES("attributes deletion"), + DELETE_TELEMETRY("telemetry deletion"), + DELETE_EVENTS("events deletion"), + UNASSIGN_ALARMS("alarms unassigning"), + DELETE_ENTITY_ALARMS("entity alarms deletion"); + + private final String description; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java index 5a8fd87933f..ec66a3d53a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java @@ -30,6 +30,7 @@ public enum NotificationType { RULE_NODE, RATE_LIMITS, EDGE_CONNECTION, - EDGE_COMMUNICATION_FAILURE + EDGE_COMMUNICATION_FAILURE, + TASK_PROCESSING_FAILURE } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java new file mode 100644 index 00000000000..cbee861c049 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.info; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +import static org.thingsboard.server.common.data.util.CollectionsUtil.mapOf; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class TaskProcessingFailureNotificationInfo implements RuleOriginatedNotificationInfo { + + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private String taskDescription; + private String error; + private int attempt; + + @Override + public Map getTemplateData() { + return mapOf( + "tenantId", tenantId.toString(), + "entityType", entityId.getEntityType().getNormalName(), + "entityId", entityId.getId().toString(), + "taskType", taskType.getDescription(), + "taskDescription", taskDescription, + "error", error, + "attempt", String.valueOf(attempt) + ); + } + + @Override + public TenantId getAffectedTenantId() { + return tenantId; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java new file mode 100644 index 00000000000..9ccfac6aff5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.rule.trigger; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; + +import java.util.concurrent.TimeUnit; + +@Data +@Builder +public class TaskProcessingFailureTrigger implements NotificationRuleTrigger { + + private final HousekeeperTask task; + private final int attempt; + private final Throwable error; + + @Override + public NotificationRuleTriggerType getType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + + @Override + public TenantId getTenantId() { + return task.getTenantId(); + } + + @Override + public EntityId getOriginatorEntityId() { + return task.getEntityId(); + } + + @Override + public boolean deduplicate() { + return true; + } + + @Override + public String getDeduplicationKey() { + return String.join(":", NotificationRuleTrigger.super.getDeduplicationKey(), task.getTaskType().toString()); + } + + @Override + public long getDefaultDeduplicationDuration() { + return TimeUnit.MINUTES.toMillis(30); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java index 15a5e592551..1621ec81200 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java @@ -38,6 +38,7 @@ @Type(value = RateLimitsNotificationRuleTriggerConfig.class, name = "RATE_LIMITS"), @Type(value = EdgeConnectionNotificationRuleTriggerConfig.class, name = "EDGE_CONNECTION"), @Type(value = EdgeCommunicationFailureNotificationRuleTriggerConfig.class, name = "EDGE_COMMUNICATION_FAILURE"), + @Type(value = TaskProcessingFailureNotificationRuleTriggerConfig.class, name = "TASK_PROCESSING_FAILURE") }) public interface NotificationRuleTriggerConfig extends Serializable { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java index 8469fac752f..8846ae284d9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java @@ -31,7 +31,8 @@ public enum NotificationRuleTriggerType { NEW_PLATFORM_VERSION(false), ENTITIES_LIMIT(false), API_USAGE_LIMIT(false), - RATE_LIMITS(false); + RATE_LIMITS(false), + TASK_PROCESSING_FAILURE(false); private final boolean tenantLevel; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java similarity index 61% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java index 51fb9ab9268..73f0517238d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java @@ -13,13 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.notification.rule.trigger.config; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class TaskProcessingFailureNotificationRuleTriggerConfig implements NotificationRuleTriggerConfig { + + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } -public enum HousekeeperTaskType { - DELETE_ENTITIES, - DELETE_ATTRIBUTES, - DELETE_TELEMETRY, // maybe divide into latest and ts kv history? - DELETE_EVENTS, - UNASSIGN_ALARMS, - DELETE_ENTITY_ALARMS } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 5bdcd341b86..b81b6dc5e69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -21,12 +21,13 @@ import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; +import java.util.Optional; import java.util.UUID; @Component @@ -34,7 +35,7 @@ @Slf4j public class CleanUpService { - private final HousekeeperService housekeeperService; + private final Optional housekeeperService; private final RelationService relationService; @TransactionalEventListener(fallbackExecution = true) @@ -44,24 +45,31 @@ public void handleEntityDeletionEvent(DeleteEntityEvent event) { log.trace("[{}] Handling entity deletion event: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { - housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + housekeeperService.ifPresent(housekeeperService -> { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + }); } } public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + housekeeperService.ifPresent(housekeeperService -> { + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + }); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { UUID tasksKey = UUID.randomUUID(); // so that all tasks are pushed to single partition to be processed synchronously - for (EntityType entityType : entityTypes) { - housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); - } + // todo: just use tenantId as key in the impl + housekeeperService.ifPresent(housekeeperService -> { + for (EntityType entityType : entityTypes) { + housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); + } + }); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java index 5e55318953c..1bbbb25df1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -15,13 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import java.util.UUID; public interface HousekeeperService { - void submitTask(HousekeeperTask task); + default void submitTask(HousekeeperTask task) { + submitTask(task.getEntityId().getId(), task); + } // tasks with the same key will be pushed to the same partition and thus processed synchronously void submitTask(UUID key, HousekeeperTask task); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java index 1bdc835b291..a37d37e8491 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java @@ -173,17 +173,15 @@ public void createDefaultNotificationConfigs(TenantId tenantId) { defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForTenant, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.newPlatformVersion, sysAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, tenantAdmins.getId()); return; } @@ -206,19 +204,19 @@ public void createDefaultNotificationConfigs(TenantId tenantId) { @Override public void updateDefaultNotificationConfigs(TenantId tenantId) { if (tenantId.isSysTenantId()) { - if (notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, - List.of(NotificationType.RATE_LIMITS), new PageLink(1)).getTotalElements() > 0) { - return; - } - NotificationTarget sysAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.SYSTEM_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "System administrators", new SystemAdministratorsFilter(), "All system administrators")); NotificationTarget affectedTenantAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.AFFECTED_TENANT_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "Affected tenant's administrators", new AffectedTenantAdministratorsFilter(), "")); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + if (!isNotificationConfigured(tenantId, NotificationType.RATE_LIMITS)) { + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + } + if (!isNotificationConfigured(tenantId, NotificationType.TASK_PROCESSING_FAILURE)) { + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, sysAdmins.getId()); + } } else { var requiredNotificationTypes = List.of(NotificationType.EDGE_CONNECTION, NotificationType.EDGE_COMMUNICATION_FAILURE); var existingNotificationTypes = notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes( @@ -252,6 +250,10 @@ tenantId, requiredNotificationTypes, new PageLink(1)) } } + private boolean isNotificationConfigured(TenantId tenantId, NotificationType... notificationTypes) { + return notificationTemplateService.countNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, List.of(notificationTypes)) > 0; + } + private NotificationTarget createTarget(TenantId tenantId, String name, UsersFilter filter, String description) { NotificationTarget target = new NotificationTarget(); target.setTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index 6cf8239f478..bec6c3d61cd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -69,6 +69,11 @@ public PageData findNotificationTemplatesByTenantIdAndNoti return notificationTemplateDao.findByTenantIdAndNotificationTypesAndPageLink(tenantId, notificationTypes, pageLink); } + @Override + public int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes) { + return notificationTemplateDao.countByTenantIdAndNotificationTypes(tenantId, notificationTypes); + } + @Override public void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id) { if (notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index fd17618bd1a..4f617c2b9d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; import org.thingsboard.server.common.data.notification.rule.trigger.config.RateLimitsNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.rule.trigger.config.RuleEngineComponentLifecycleEventNotificationRuleTriggerConfig; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.common.data.notification.template.NotificationTemplateConfig; import org.thingsboard.server.common.data.notification.template.WebDeliveryMethodNotificationTemplate; @@ -358,6 +359,19 @@ public class DefaultNotifications { .build()) .build(); + public static final DefaultNotification taskProcessingFailure = DefaultNotification.builder() + .name("Task processing failure notification") + .type(NotificationType.TASK_PROCESSING_FAILURE) + .subject("Failed to process ${taskType}") + .text("Failed to process ${taskDescription} for tenant ${tenantId}: ${error}") + .icon("warning").color(YELLOW_COLOR) + .rule(DefaultRule.builder() + .name("Task processing failure") + .triggerConfig(TaskProcessingFailureNotificationRuleTriggerConfig.builder().build()) + .description("Send notification to system admins when task processing fails") + .build()) + .build(); + public static final DefaultNotification jwtSigningKeyIssue = DefaultNotification.builder() .name("JWT Signing Key issue notification") .type(NotificationType.GENERAL) diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java index d9a80ce3f5e..136bc4d4720 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java @@ -30,6 +30,8 @@ public interface NotificationTemplateDao extends Dao, Expo PageData findByTenantIdAndNotificationTypesAndPageLink(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void removeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java index 8ee6c3df3a0..8950ab1e358 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java @@ -53,6 +53,11 @@ public PageData findByTenantIdAndNotificationTypesAndPageL notificationTypes, pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public int countByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes) { + return notificationTemplateRepository.countByTenantIdAndNotificationTypes(tenantId.getId(), notificationTypes); + } + @Override public void removeByTenantId(TenantId tenantId) { notificationTemplateRepository.deleteByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java index b3e416a491d..bb45c10d065 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java @@ -42,6 +42,11 @@ Page findByTenantIdAndNotificationTypesAndSearchText @Param("searchText") String searchText, Pageable pageable); + @Query("SELECT count(t) FROM NotificationTemplateEntity t WHERE t.tenantId = :tenantId AND " + + "t.notificationType IN :notificationTypes") + int countByTenantIdAndNotificationTypes(@Param("tenantId") UUID tenantId, + @Param("notificationTypes") List notificationTypes); + @Transactional @Modifying @Query("DELETE FROM NotificationTemplateEntity t WHERE t.tenantId = :tenantId") diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html index 5b477d0345f..f5a2ddba17a 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html @@ -581,6 +581,19 @@

{{ dialogTitle | translate }}

+ + + {{ 'notification.task-processing-failure-trigger-settings' | translate }} +
+
+ + notification.description + + +
+
+ diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts index 89488b75173..bc6935b4e7d 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts @@ -101,6 +101,7 @@ export class RuleNotificationDialogComponent extends rateLimitsTemplateForm: FormGroup; edgeCommunicationFailureTemplateForm: FormGroup; edgeConnectionTemplateForm: FormGroup; + taskProcessingFailureTemplateForm: FormGroup; triggerType = TriggerType; triggerTypes: TriggerType[]; @@ -337,6 +338,12 @@ export class RuleNotificationDialogComponent extends }) }); + this.taskProcessingFailureTemplateForm = this.fb.group({ + triggerConfig: this.fb.group({ + taskTypes: [] + }) + }); + this.triggerTypeFormsMap = new Map([ [TriggerType.ALARM, this.alarmTemplateForm], [TriggerType.ALARM_COMMENT, this.alarmCommentTemplateForm], @@ -349,7 +356,8 @@ export class RuleNotificationDialogComponent extends [TriggerType.NEW_PLATFORM_VERSION, this.newPlatformVersionTemplateForm], [TriggerType.RATE_LIMITS, this.rateLimitsTemplateForm], [TriggerType.EDGE_COMMUNICATION_FAILURE, this.edgeCommunicationFailureTemplateForm], - [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm] + [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm], + [TriggerType.TASK_PROCESSING_FAILURE, this.taskProcessingFailureTemplateForm] ]); if (data.isAdd || data.isCopy) { @@ -488,7 +496,8 @@ export class RuleNotificationDialogComponent extends TriggerType.ENTITIES_LIMIT, TriggerType.API_USAGE_LIMIT, TriggerType.NEW_PLATFORM_VERSION, - TriggerType.RATE_LIMITS + TriggerType.RATE_LIMITS, + TriggerType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts index c0ae4f1abbf..ccbcb6bf223 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts @@ -179,7 +179,8 @@ export class TemplateNotificationDialogComponent NotificationType.ENTITIES_LIMIT, NotificationType.API_USAGE_LIMIT, NotificationType.NEW_PLATFORM_VERSION, - NotificationType.RATE_LIMITS + NotificationType.RATE_LIMITS, + NotificationType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/shared/models/notification.models.ts b/ui-ngx/src/app/shared/models/notification.models.ts index 2cc7d8547bb..0c5e1c5a90c 100644 --- a/ui-ngx/src/app/shared/models/notification.models.ts +++ b/ui-ngx/src/app/shared/models/notification.models.ts @@ -523,7 +523,8 @@ export enum NotificationType { RULE_NODE = 'RULE_NODE', RATE_LIMITS = 'RATE_LIMITS', EDGE_CONNECTION = 'EDGE_CONNECTION', - EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE' + EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE', + TASK_PROCESSING_FAILURE = 'TASK_PROCESSING_FAILURE' } export const NotificationTypeIcons = new Map([ @@ -535,6 +536,7 @@ export const NotificationTypeIcons = new Map([ [NotificationType.RULE_ENGINE_COMPONENT_LIFECYCLE_EVENT, 'settings_ethernet'], [NotificationType.ENTITIES_LIMIT, 'data_thresholding'], [NotificationType.API_USAGE_LIMIT, 'insert_chart'], + [NotificationType.TASK_PROCESSING_FAILURE, 'warning'] ]); export const AlarmSeverityNotificationColors = new Map( @@ -646,7 +648,13 @@ export const NotificationTemplateTypeTranslateMap = new Map([ @@ -676,7 +685,8 @@ export const TriggerTypeTranslationMap = new Map([ [TriggerType.NEW_PLATFORM_VERSION, 'notification.trigger.new-platform-version'], [TriggerType.RATE_LIMITS, 'notification.trigger.rate-limits'], [TriggerType.EDGE_CONNECTION, 'notification.trigger.edge-connection'], - [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'] + [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'], + [TriggerType.TASK_PROCESSING_FAILURE, 'notification.trigger.task-processing-failure'] ]); export interface NotificationUserSettings { diff --git a/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md new file mode 100644 index 00000000000..0bcf107298e --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md @@ -0,0 +1,46 @@ +#### Task processing failure notification templatization + +
+
+ +Notification subject and message fields support templatization. +The list of available templatization parameters depends on the template type. +See the available types and parameters below: + +Available template parameters: + +* `taskType` - the task type, e.g. 'telemetry deletion'; +* `taskDescription` - the task description, e.g. 'telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325'; +* `error` - the error stacktrace +* `tenantId` - the tenant id; +* `entityType` - the type of the entity to which the task is related; +* `entityId` - the id of the entity to which the task is related; +* `attempt` - the number of attempts processing the task + +Parameter names must be wrapped using `${...}`. For example: `${entityType}`. +You may also modify the value of the parameter with one of the suffixes: + +* `upperCase`, for example - `${entityType:upperCase}` +* `lowerCase`, for example - `${entityType:lowerCase}` +* `capitalize`, for example - `${entityType:capitalize}` + +
+ +##### Examples + +Let's assume that telemetry deletion failed for some device. +The following template: + +```text +Failed to process ${taskType} for ${entityType:lowerCase} ${entityId} +{:copy-code} +``` + +will be transformed to: + +```text +Failed to process telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325 +``` + +
+
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 04280876203..f94a2fffd31 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3241,6 +3241,7 @@ "api-usage-trigger-settings": "API usage trigger settings", "new-platform-version-trigger-settings": "New platform version trigger settings", "rate-limits-trigger-settings": "Exceeded rate limits trigger settings", + "task-processing-failure-trigger-settings": "Task processing failure trigger settings", "at-least-one-should-be-selected": "At least one should be selected", "basic-settings": "Basic settings", "button-text": "Button text", @@ -3450,7 +3451,8 @@ "new-platform-version": "New platform version", "rate-limits": "Exceeded rate limits", "edge-communication-failure": "Edge communication failure", - "edge-connection": "Edge connection" + "edge-connection": "Edge connection", + "task-processing-failure": "Task processing failure" }, "templates": "Templates", "notification-templates": "Notifications / Templates", @@ -3473,6 +3475,7 @@ "rate-limits": "Exceeded rate limits", "edge-connection": "Edge connection", "edge-communication-failure": "Edge communication failure", + "task-processing-failure": "Task processing failure", "trigger": "Trigger", "trigger-required": "Trigger is required" }, From e1f8f519006d313b4124d92de65a207f620fc26f Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 1 Mar 2024 14:45:51 +0200 Subject: [PATCH 018/304] UI: Color picker design improvement --- .../components/color-input.component.ts | 8 +- .../color-picker-panel.component.html | 26 ++++--- .../color-picker-panel.component.scss | 17 ++++- .../color-picker-panel.component.ts | 16 +++- .../color-picker/color-picker.component.html | 57 +++++++------- .../color-picker/color-picker.component.scss | 49 +++++++++++- .../color-picker/color-picker.component.ts | 4 +- .../color-picker/hex-input.component.html | 36 +++++++++ .../color-picker/hex-input.component.scss | 48 ++++++++++++ .../color-picker/hex-input.component.ts | 75 +++++++++++++++++++ .../dialog/color-picker-dialog.component.html | 8 +- .../dialog/color-picker-dialog.component.scss | 28 +++++++ .../shared/components/popover.component.ts | 3 +- .../app/shared/components/popover.service.ts | 9 ++- ui-ngx/src/app/shared/shared.module.ts | 16 ++-- 15 files changed, 333 insertions(+), 67 deletions(-) create mode 100644 ui-ngx/src/app/shared/components/color-picker/hex-input.component.html create mode 100644 ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss create mode 100644 ui-ngx/src/app/shared/components/color-picker/hex-input.component.ts diff --git a/ui-ngx/src/app/shared/components/color-input.component.ts b/ui-ngx/src/app/shared/components/color-input.component.ts index ad772ba861b..706db506770 100644 --- a/ui-ngx/src/app/shared/components/color-input.component.ts +++ b/ui-ngx/src/app/shared/components/color-input.component.ts @@ -189,13 +189,13 @@ export class ColorInputComponent extends PageComponent implements OnInit, Contro this.popoverService.hidePopover(trigger); } else { const colorPickerPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, ColorPickerPanelComponent, ['leftTopOnly', 'leftOnly', 'leftBottomOnly'], true, null, + this.viewContainerRef, ColorPickerPanelComponent, ['left'], true, null, { color: this.colorFormGroup.get('color').value, - colorClearButton: this.colorClearButton + colorClearButton: this.colorClearButton, + colorCancelButton: true }, - {}, - {}, {}, true); + {}, {}, {}, false, () => {}, {padding: '12px 4px 12px 12px'}); colorPickerPopover.tbComponentRef.instance.popover = colorPickerPopover; colorPickerPopover.tbComponentRef.instance.colorSelected.subscribe((color) => { colorPickerPopover.hide(); diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.html b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.html index 40e66538670..6ddae212e11 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.html +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.html @@ -16,8 +16,7 @@ -->
-
color.color
- +
- + +
+ + +
diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss index d25f6b9d85a..fc71d698220 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss @@ -13,11 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@import "../scss/constants"; + .tb-color-picker-panel { - width: 328px; + width: 342px; display: flex; flex-direction: column; - gap: 16px; + max-height: calc(100vh - 24px); + min-height: 100%; + @media #{$mat-sm} { + width: 578px; + } .tb-color-picker-title { font-size: 16px; font-weight: 500; @@ -25,12 +31,17 @@ letter-spacing: 0.25px; color: rgba(0, 0, 0, 0.87); } + .tb-color-picker { + padding-right: 4px; + overflow-y: scroll; + } .tb-color-picker-panel-buttons { height: 60px; + padding: 8px 8px 0 0; display: flex; flex-direction: row; gap: 16px; - justify-content: flex-end; + justify-content: space-between; align-items: flex-end; } } diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.ts b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.ts index 8ed2d304aed..443dd6967f9 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.ts +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.ts @@ -38,12 +38,19 @@ export class ColorPickerPanelComponent extends PageComponent implements OnInit { @coerceBoolean() colorClearButton = false; + @Input() + @coerceBoolean() + colorCancelButton = false; + @Input() popover: TbPopoverComponent; @Output() colorSelected = new EventEmitter(); + @Output() + colorCancelDialog = new EventEmitter(); + colorPickerControl: UntypedFormControl; constructor(protected store: Store) { @@ -61,4 +68,11 @@ export class ColorPickerPanelComponent extends PageComponent implements OnInit { clearColor() { this.colorSelected.emit(null); } -} + + cancelColor() { + if (this.popover) { + this.popover.hide(); + } else { + this.colorCancelDialog.emit(); + } + }} diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html index 7940f1ec73d..888c5b0b0bc 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html @@ -15,35 +15,38 @@ limitations under the License. --> - - -
- - -
- - -
-
- -
- - HEX - RGBA - HSLA - -
- - - +
+ + +
+
+ + +
+ + +
+
+ +
+ + HEX + RGBA + HSLA + +
+ + + +
+
- +
diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss index 9aca5809943..ef22ac203f1 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@import "../scss/constants"; + :host { width: 100%; display: flex; @@ -20,10 +22,34 @@ gap: 32px; overflow: auto; + .color-input-container { + display: flex; + flex-direction: column; + gap: 32px; + @media #{$mat-sm} { + flex-direction: row; + gap: 12px; + } + } + + .control-input-container { + display: flex; + flex-direction: column; + gap: 32px; + @media #{$mat-sm} { + width: 100%; + justify-content: center; + } + } + .saturation-component { + width: 100%; height: 238px; - min-height: 80px; + min-height: 160px; border-radius: 8px; + @media #{$mat-sm} { + max-height: 160px; + } } .control-component { @@ -54,7 +80,9 @@ } .color-input-block { + height: 56px; display: flex; + align-items: center; gap: 20px; .presentation-select { @@ -74,8 +102,13 @@ .color-presets-block { .color-presets-component { display: flex; - flex-direction: column; - gap: 12px; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-between; + gap: 8px; + @media #{$mat-xs} { + flex-direction: column; + } } } } @@ -111,13 +144,21 @@ .color-presets-component { .presets-row { - gap: 10px; + gap: 8px; justify-content: space-between; } color-preset { height: 20px; width: 20px; border-radius: 4px; + @media #{$mat-xs} { + height: 40px; + width: 48px; + } + @media #{$mat-sm} { + height: 40px; + width: 40px; + } } } } diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.ts b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.ts index 5cca12a9683..100d1177cfe 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.ts +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.ts @@ -31,8 +31,8 @@ export enum ColorType { } const colorPresetsHex = - ['#435B63', '#F44336', '#E89623', '#F5DD00', '#8BC34A', '#4CAF50', '#009688', '#048AD3', '#673AB7', '#9C27B0', '#E91E63', - '#A1ADB1', '#F9A19B', '#FFD190', '#FFF59D', '#C5E1A4', '#A5D7A7', '#80CBC3', '#81C4E9', '#B39CDB', '#CD93D7', '#F48FB1']; + ['#435B63', '#F44336', '#E89623', '#F5DD00', '#8BC34A', '#4CAF50', '#009688', '#048AD3', '#673AB7', '#9C27B0', '#E91E63', '#6F113A', + '#A1ADB1', '#F9A19B', '#FFD190', '#FFF59D', '#C5E1A4', '#A5D7A7', '#80CBC3', '#81C4E9', '#B39CDB', '#CD93D7', '#F48FB1', '#BC91A4']; @Component({ selector: `tb-color-picker`, diff --git a/ui-ngx/src/app/shared/components/color-picker/hex-input.component.html b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.html new file mode 100644 index 00000000000..c3faa497cef --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.html @@ -0,0 +1,36 @@ + +
+ + {{prefixValue}} + + + + + + + % + +
diff --git a/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss new file mode 100644 index 00000000000..0f5125874c2 --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +:host { + .hex-input-container { + display: flex; + gap: 8px; + } + .hex-input { + max-width: 190px; + } + .alpha-input { + min-width: 60px; + max-width: 60px; + } + + ::ng-deep { + .mdc-text-field--filled, .mat-mdc-form-field-focus-overlay { + &:before { + background-color: transparent !important; + } + } + .mat-mdc-form-field-icon-prefix, .mdc-line-ripple, .copy-button { + opacity: 0.4; + } + .alpha-input { + .mat-mdc-text-field-wrapper { + padding-left: 0; + } + } + + } +} + + diff --git a/ui-ngx/src/app/shared/components/color-picker/hex-input.component.ts b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.ts new file mode 100644 index 00000000000..fcd91a23975 --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.ts @@ -0,0 +1,75 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectionStrategy, Component, EventEmitter, Input, Output } from '@angular/core'; +import { Color } from '@iplab/ngx-color-picker'; + +@Component({ + selector: `tb-hex-input`, + templateUrl: `./hex-input.component.html`, + styleUrls: ['./hex-input.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class HexInputComponent { + + @Input() + public color: Color; + + @Output() + public colorChange = new EventEmitter(false); + + @Input() + public labelVisible = false; + + @Input() + public prefixValue = '#'; + + public get value() { + return this.color ? this.color.toHexString(this.color.getRgba().alpha < 1).replace('#', '') : ''; + } + + public get copyColor() { + return this.prefixValue + this.value; + } + + public get hueValue(): string { + return this.color ? Math.round(this.color.getRgba().alpha * 100).toString() : ''; + } + + public onHueInputChange(event: KeyboardEvent, inputValue: string): void { + const color = this.color.getRgba(); + const alpha = +inputValue / 100; + if (color.getAlpha() !== alpha) { + const newColor = new Color().setRgba(color.red, color.green, color.blue, alpha).toHexString(true); + this.colorChange.emit(new Color(newColor)); + } + } + + public onInputChange(event: KeyboardEvent, inputValue: string): void { + const value = inputValue.toLowerCase(); + if ( + ((event.keyCode === 13 || event.key.toLowerCase() === 'enter') && value.length === 3) + || value.length === 6 || value.length === 8 + ) { + const hex = parseInt(value, 16); + const hexStr = hex.toString(16); + if (hexStr.padStart(value.length, '0') === value && this.value !== value) { + const newColor = new Color(`#${value}`); + this.colorChange.emit(newColor); + } + } + } +} diff --git a/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.html index 3d37d83c15c..35b560bb3a9 100644 --- a/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.html @@ -16,14 +16,10 @@ -->
-
diff --git a/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.scss b/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.scss index 2a4ec55f7d0..e5b24d78e17 100644 --- a/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.scss +++ b/ui-ngx/src/app/shared/components/dialog/color-picker-dialog.component.scss @@ -13,10 +13,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@import "../scss/constants"; + :host { .tb-close-button { position: absolute; top: 6px; right: 6px; } + ::ng-deep { + .mat-mdc-dialog-content { + max-height: 100%; + overflow: hidden; + padding: 12px 4px 12px 12px !important; + } + .tb-color-picker-panel { + @media #{$mat-sm} { + width: 342px; + .color-input-container { + flex-direction: column !important; + } + .color-presets-component color-preset { + width: 48px; + } + } + + @media #{$mat-xs} { + width: 100%; + .hex-input { + flex: 1; + max-width: 100% !important; + } + } + } + } } diff --git a/ui-ngx/src/app/shared/components/popover.component.ts b/ui-ngx/src/app/shared/components/popover.component.ts index de2c3282041..f0aa4981afd 100644 --- a/ui-ngx/src/app/shared/components/popover.component.ts +++ b/ui-ngx/src/app/shared/components/popover.component.ts @@ -340,7 +340,7 @@ export class TbPopoverDirective implements OnChanges, OnDestroy, AfterViewInit {