diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 4a60b69165..fd38c48617 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -1081,7 +1081,7 @@ private void saveRpcResponseToCloudQueue(ToDeviceRpcRequestActorMsg msg, int req try { systemContext.getCloudEventService().saveCloudEventAsync(tenantId, CloudEventType.DEVICE, EdgeEventActionType.RPC_CALL, - deviceId, body, 0L).get(); + deviceId, body).get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 8149702e6b..41b23a8d33 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -68,8 +68,6 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; -import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cloud.CloudEventType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.domain.Domain; @@ -134,9 +132,9 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.EdgeSettingsService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.device.ClaimDevicesService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; @@ -174,16 +172,12 @@ import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.user.TbUserSettingsService; -import org.thingsboard.server.service.ota.OtaPackageStateService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -303,9 +297,6 @@ public abstract class BaseController { @Autowired protected AuditLogService auditLogService; - @Autowired - protected DeviceStateService deviceStateService; - @Autowired protected EntityViewService entityViewService; @@ -316,10 +307,10 @@ public abstract class BaseController { protected AttributesService attributesService; @Autowired - protected ClaimDevicesService claimDevicesService; + protected CloudEventService cloudEventService; @Autowired - protected CloudEventService cloudEventService; + protected EdgeSettingsService edgeSettingsService; @Autowired protected PartitionService partitionService; @@ -330,9 +321,6 @@ public abstract class BaseController { @Autowired protected OtaPackageService otaPackageService; - @Autowired - protected OtaPackageStateService otaPackageStateService; - @Autowired protected RpcService rpcService; @@ -345,9 +333,6 @@ public abstract class BaseController { @Autowired protected TbDeviceProfileCache deviceProfileCache; - @Autowired - protected TbAssetProfileCache assetProfileCache; - @Autowired(required = false) protected EdgeService edgeService; @@ -360,9 +345,6 @@ public abstract class BaseController { @Autowired protected QueueService queueService; - @Autowired - protected EntitiesVersionControlService vcService; - @Autowired protected ExportableEntitiesService entitiesService; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index e01b453353..b1eabc32e4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -56,7 +56,6 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest; import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse; -import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -74,7 +73,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -473,7 +471,7 @@ public List getEdgeTypes() throws ThingsboardException, Execution @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping(value = "/edge/sync/{edgeId}") public DeferredResult syncEdge(@Parameter(description = EDGE_ID_PARAM_DESCRIPTION, required = true) - @PathVariable("edgeId") String strEdgeId) throws ThingsboardException { + @PathVariable("edgeId") String strEdgeId) throws ThingsboardException { checkParameter("edgeId", strEdgeId); final DeferredResult response = new DeferredResult<>(); if (isEdgesEnabled() && edgeRpcServiceOpt.isPresent()) { @@ -565,7 +563,7 @@ public EdgeSettings getEdgeSettings() throws ThingsboardException { try { SecurityUser user = getCurrentUser(); TenantId tenantId = user.getTenantId(); - return checkNotNull(cloudEventService.findEdgeSettings(tenantId)); + return checkNotNull(edgeSettingsService.findEdgeSettings(tenantId)); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/BaseCloudManagerService.java b/application/src/main/java/org/thingsboard/server/service/cloud/BaseCloudManagerService.java new file mode 100644 index 0000000000..1a45a2da28 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cloud/BaseCloudManagerService.java @@ -0,0 +1,758 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cloud; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.edge.rpc.EdgeRpcClient; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.cloud.CloudEvent; +import org.thingsboard.server.common.data.cloud.CloudEventType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeSettings; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.EdgeSettingsService; +import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; +import org.thingsboard.server.gen.edge.v1.EdgeVersion; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import org.thingsboard.server.service.cloud.rpc.CloudEventStorageSettings; +import org.thingsboard.server.service.cloud.rpc.processor.AlarmCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.AssetCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.AssetProfileCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.CustomerCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.DashboardCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.DeviceCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.DeviceProfileCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.EdgeCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.EntityViewCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.RelationCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.ResourceCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.TelemetryCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.TenantCloudProcessor; +import org.thingsboard.server.service.cloud.rpc.processor.WidgetBundleCloudProcessor; +import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.thingsboard.server.service.edge.rpc.EdgeGrpcSession.RATE_LIMIT_REACHED; + +@Slf4j +public abstract class BaseCloudManagerService { + + protected static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; + protected static final String QUEUE_SEQ_ID_OFFSET_ATTR_KEY = "queueSeqIdOffset"; + protected static final String QUEUE_TS_KV_START_TS_ATTR_KEY = "queueTsKvStartTs"; + protected static final String QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY = "queueTsKvSeqIdOffset"; + + private static final int MAX_SEND_UPLINK_ATTEMPTS = 3; + + @Value("${cloud.routingKey}") + private String routingKey; + + @Value("${cloud.secret}") + private String routingSecret; + + @Value("${cloud.reconnect_timeout}") + private long reconnectTimeoutMs; + + @Value("${cloud.uplink_pack_timeout_sec:60}") + private long uplinkPackTimeoutSec; + + @Autowired + private EdgeService edgeService; + + @Autowired + private AttributesService attributesService; + + @Autowired + protected CloudEventStorageSettings cloudEventStorageSettings; + + @Autowired + private TelemetrySubscriptionService tsSubService; + + @Autowired + private DownlinkMessageService downlinkMessageService; + + @Autowired + private EdgeRpcClient edgeRpcClient; + + @Autowired + private EdgeCloudProcessor edgeCloudProcessor; + + @Autowired + private TenantCloudProcessor tenantProcessor; + + @Autowired + private CustomerCloudProcessor customerProcessor; + + @Autowired + private CloudEventService cloudEventService; + + @Autowired + private EdgeSettingsService edgeSettingsService; + + @Autowired + private ConfigurableApplicationContext context; + + @Autowired + private RelationCloudProcessor relationProcessor; + + @Autowired + private DeviceCloudProcessor deviceProcessor; + + @Autowired + private DeviceProfileCloudProcessor deviceProfileProcessor; + + @Autowired + private AlarmCloudProcessor alarmProcessor; + + @Autowired + private TelemetryCloudProcessor telemetryProcessor; + + @Autowired + private WidgetBundleCloudProcessor widgetBundleProcessor; + + @Autowired + private EntityViewCloudProcessor entityViewProcessor; + + @Autowired + private DashboardCloudProcessor dashboardProcessor; + + @Autowired + private AssetCloudProcessor assetProcessor; + + @Autowired + private AssetProfileCloudProcessor assetProfileProcessor; + + @Autowired + private ResourceCloudProcessor resourceCloudProcessor; + + @Autowired + private DbCallbackExecutorService dbCallbackExecutorService; + + @Autowired(required = false) + private CloudEventMigrationService cloudEventMigrationService; + + private ScheduledExecutorService uplinkExecutor; + private ScheduledFuture sendUplinkFuture; + private ScheduledExecutorService shutdownExecutor; + + private EdgeSettings currentEdgeSettings; + protected TenantId tenantId; + private CustomerId customerId; + + private final ConcurrentMap pendingMsgMap = new ConcurrentHashMap<>(); + private CountDownLatch latch; + private SettableFuture sendUplinkFutureResult; + + protected volatile boolean initialized; + protected volatile boolean isGeneralProcessInProgress = false; + private volatile boolean sendingInProgress = false; + private volatile boolean syncInProgress = false; + private volatile boolean isRateLimitViolated = false; + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + if (validateRoutingKeyAndSecret()) { + log.info("Starting Cloud Edge service"); + edgeRpcClient.connect(routingKey, routingSecret, + this::onUplinkResponse, + this::onEdgeUpdate, + this::onDownlink, + this::scheduleReconnect); + uplinkExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("cloud-manager-uplink")); + launchUplinkProcessing(); + } + } + + protected abstract void launchUplinkProcessing(); + + protected void resetQueueOffset() { + updateQueueStartTsSeqIdOffset(tenantId, QUEUE_START_TS_ATTR_KEY, QUEUE_SEQ_ID_OFFSET_ATTR_KEY, System.currentTimeMillis(), 0L); + updateQueueStartTsSeqIdOffset(tenantId, QUEUE_TS_KV_START_TS_ATTR_KEY, QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY, System.currentTimeMillis(), 0L); + } + + protected void updateQueueStartTsSeqIdOffset(TenantId tenantId, String attrStartTsKey, String attrSeqIdKey, Long startTs, Long seqIdOffset) { + log.trace("updateQueueStartTsSeqIdOffset [{}][{}][{}][{}]", attrStartTsKey, attrSeqIdKey, startTs, seqIdOffset); + List attributes = Arrays.asList( + new BaseAttributeKvEntry(new LongDataEntry(attrStartTsKey, startTs), System.currentTimeMillis()), + new BaseAttributeKvEntry(new LongDataEntry(attrSeqIdKey, seqIdOffset), System.currentTimeMillis()) + ); + try { + attributesService.save(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attributes).get(); + } catch (Exception e) { + log.error("Failed to update queueStartTsSeqIdOffset [{}][{}]", attrStartTsKey, attrSeqIdKey, e); + } + } + + protected void destroy() throws InterruptedException { + if (shutdownExecutor != null) { + shutdownExecutor.shutdown(); + } + + updateConnectivityStatus(false); + + String edgeId = currentEdgeSettings != null ? currentEdgeSettings.getEdgeId() : ""; + log.info("[{}] Starting destroying process", edgeId); + try { + edgeRpcClient.disconnect(false); + } catch (Exception e) { + log.error("Exception during disconnect", e); + } + + if (uplinkExecutor != null) { + uplinkExecutor.shutdownNow(); + } + log.info("[{}] Destroy was successful", edgeId); + } + + protected void processUplinkMessages(TimePageLink pageLink, Long queueSeqIdStart, String queueStartTsAttrKey, String queueSeqIdAttrKey, boolean isGeneralMsg, CloudEventFinder finder) { + try { + if (isGeneralMsg) { + isGeneralProcessInProgress = true; + } + PageData cloudEvents; + boolean isInterrupted; + do { + cloudEvents = finder.find(tenantId, queueSeqIdStart, null, pageLink); + if (cloudEvents.getData().isEmpty()) { + log.info("seqId column of table started new cycle"); + cloudEvents = findCloudEventsFromBeginning(tenantId, pageLink, finder); + } + isInterrupted = processCloudEvents(cloudEvents.getData(), isGeneralMsg).get(); + if (!isInterrupted && cloudEvents.getTotalElements() > 0) { + CloudEvent latestCloudEvent = cloudEvents.getData().get(cloudEvents.getData().size() - 1); + try { + Long newStartTs = Uuids.unixTimestamp(latestCloudEvent.getUuidId()); + updateQueueStartTsSeqIdOffset(tenantId, queueStartTsAttrKey, queueSeqIdAttrKey, newStartTs, latestCloudEvent.getSeqId()); + log.debug("Queue offset was updated [{}][{}][{}]", latestCloudEvent.getUuidId(), newStartTs, latestCloudEvent.getSeqId()); + } catch (Exception e) { + log.error("Failed to update queue offset [{}]", latestCloudEvent); + } + } + if (!isInterrupted) { + pageLink = pageLink.nextPageLink(); + } + if (!isGeneralMsg && isGeneralProcessInProgress) { + break; + } + log.trace("processUplinkMessages state isInterrupted={},total={},hasNext={},isGeneralMsg={},isGeneralProcessInProgress={}", + isInterrupted, cloudEvents.getTotalElements(), cloudEvents.hasNext(), isGeneralMsg, isGeneralProcessInProgress); + } while (isInterrupted || cloudEvents.hasNext()); + } catch (Exception e) { + log.error("Failed to process cloud event messages handling!", e); + } finally { + if (isGeneralMsg) { + isGeneralProcessInProgress = false; + } + } + } + + protected TimePageLink newCloudEventsAvailable(TenantId tenantId, Long queueSeqIdStart, String key, CloudEventFinder finder) { + try { + long queueStartTs = getLongAttrByKey(tenantId, key).get(); + long queueEndTs = queueStartTs > 0 ? queueStartTs + TimeUnit.DAYS.toMillis(1) : System.currentTimeMillis(); + log.trace("newCloudEventsAvailable, queueSeqIdStart = {}, key = {}, queueStartTs = {}, queueEndTs = {}", + queueSeqIdStart, key, queueStartTs, queueEndTs); + TimePageLink pageLink = new TimePageLink(cloudEventStorageSettings.getMaxReadRecordsCount(), + 0, null, null, queueStartTs, queueEndTs); + PageData cloudEvents = finder.find(tenantId, queueSeqIdStart, null, pageLink); + if (cloudEvents.getData().isEmpty()) { + if (queueSeqIdStart > cloudEventStorageSettings.getMaxReadRecordsCount()) { + // check if new cycle started (seq_id starts from '1') + cloudEvents = findCloudEventsFromBeginning(tenantId, pageLink, finder); + if (cloudEvents.getData().stream().anyMatch(ce -> ce.getSeqId() == 1)) { + log.info("newCloudEventsAvailable: new cycle started (seq_id starts from '1')!"); + return pageLink; + } + } + while (queueEndTs < System.currentTimeMillis()) { + log.trace("newCloudEventsAvailable: queueEndTs < System.currentTimeMillis() [{}] [{}]", queueEndTs, System.currentTimeMillis()); + queueStartTs = queueEndTs; + queueEndTs = queueEndTs + TimeUnit.DAYS.toMillis(1); + pageLink = new TimePageLink(cloudEventStorageSettings.getMaxReadRecordsCount(), + 0, null, null, queueStartTs, queueEndTs); + cloudEvents = finder.find(tenantId, queueSeqIdStart, null, pageLink); + if (!cloudEvents.getData().isEmpty()) { + return pageLink; + } + } + return null; + } else { + return pageLink; + } + } catch (Exception e) { + log.warn("Failed to check newCloudEventsAvailable!", e); + return null; + } + } + + protected PageData findCloudEventsFromBeginning(TenantId tenantId, TimePageLink pageLink, CloudEventFinder finder) { + long seqIdEnd = Integer.toUnsignedLong(cloudEventStorageSettings.getMaxReadRecordsCount()); + seqIdEnd = Math.max(seqIdEnd, 50L); + return finder.find(tenantId, 0L, seqIdEnd, pageLink); + } + + protected ListenableFuture getLongAttrByKey(TenantId tenantId, String attrKey) { + ListenableFuture> future = + attributesService.find(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attrKey); + return Futures.transform(future, attributeKvEntryOpt -> { + if (attributeKvEntryOpt != null && attributeKvEntryOpt.isPresent()) { + AttributeKvEntry attributeKvEntry = attributeKvEntryOpt.get(); + return attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } else { + return 0L; + } + }, dbCallbackExecutorService); + } + + private boolean validateRoutingKeyAndSecret() { + if (StringUtils.isBlank(routingKey) || StringUtils.isBlank(routingSecret)) { + shutdownExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("cloud-manager-shutdown")); + shutdownExecutor.scheduleAtFixedRate(() -> log.error( + "Routing Key and Routing Secret must be provided! " + + "Please configure Routing Key and Routing Secret in the tb-edge.yml file " + + "or add CLOUD_ROUTING_KEY and CLOUD_ROUTING_SECRET variable to the tb-edge.conf file. " + + "ThingsBoard Edge is not going to connect to cloud!"), 0, 10, TimeUnit.SECONDS); + return false; + } + return true; + } + + private void onUplinkResponse(UplinkResponseMsg msg) { + try { + if (sendingInProgress) { + if (msg.getSuccess()) { + pendingMsgMap.remove(msg.getUplinkMsgId()); + log.debug("Msg has been processed successfully! {}", msg); + } else { + if (msg.getErrorMsg().contains(RATE_LIMIT_REACHED)) { + log.warn("Msg processing failed! {}", RATE_LIMIT_REACHED); + isRateLimitViolated = true; + } else { + log.error("Msg processing failed! Error msg: {}", msg.getErrorMsg()); + } + } + latch.countDown(); + } + } catch (Exception e) { + log.error("Can't process uplink response message [{}]", msg, e); + } + } + + private void onEdgeUpdate(EdgeConfiguration edgeConfiguration) { + try { + if (sendUplinkFuture != null) { + sendUplinkFuture.cancel(true); + sendUplinkFuture = null; + } + + if ("CE".equals(edgeConfiguration.getCloudType())) { + initAndUpdateEdgeSettings(edgeConfiguration); + } else { + new Thread(() -> { + log.error("Terminating application. CE edge can be connected only to CE server version..."); + int exitCode = -1; + int appExitCode = exitCode; + try { + appExitCode = SpringApplication.exit(context, () -> exitCode); + } finally { + System.exit(appExitCode); + } + }, "Shutdown Thread").start(); + } + } catch (Exception e) { + log.error("Can't process edge configuration message [{}]", edgeConfiguration, e); + } + } + + private void initAndUpdateEdgeSettings(EdgeConfiguration edgeConfiguration) throws Exception { + this.tenantId = new TenantId(new UUID(edgeConfiguration.getTenantIdMSB(), edgeConfiguration.getTenantIdLSB())); + + this.currentEdgeSettings = edgeSettingsService.findEdgeSettings(this.tenantId); + EdgeSettings newEdgeSettings = constructEdgeSettings(edgeConfiguration); + if (this.currentEdgeSettings == null || !this.currentEdgeSettings.getEdgeId().equals(newEdgeSettings.getEdgeId())) { + tenantProcessor.cleanUp(); + this.currentEdgeSettings = newEdgeSettings; + resetQueueOffset(); + } else { + log.trace("Using edge settings from DB {}", this.currentEdgeSettings); + } + + tenantProcessor.createTenantIfNotExists(this.tenantId); + boolean edgeCustomerIdUpdated = setOrUpdateCustomerId(edgeConfiguration); + if (edgeCustomerIdUpdated) { + customerProcessor.createCustomerIfNotExists(this.tenantId, edgeConfiguration); + } + // TODO: voba - should sync be executed in some other cases ??? + log.trace("Sending sync request, fullSyncRequired {}", this.currentEdgeSettings.isFullSyncRequired()); + edgeRpcClient.sendSyncRequestMsg(this.currentEdgeSettings.isFullSyncRequired()); + this.syncInProgress = true; + + edgeSettingsService.saveEdgeSettings(tenantId, this.currentEdgeSettings); + + saveOrUpdateEdge(tenantId, edgeConfiguration); + + updateConnectivityStatus(true); + + if (cloudEventMigrationService != null && (!cloudEventMigrationService.isMigrated() || !cloudEventMigrationService.isTsMigrated())) { + cloudEventMigrationService.migrateUnprocessedEventToKafka(); + } + + initialized = true; + } + + private boolean setOrUpdateCustomerId(EdgeConfiguration edgeConfiguration) { + EdgeId edgeId = getEdgeId(edgeConfiguration); + Edge edge = edgeService.findEdgeById(tenantId, edgeId); + CustomerId previousCustomerId = null; + if (edge != null) { + previousCustomerId = edge.getCustomerId(); + } + if (edgeConfiguration.getCustomerIdMSB() != 0 && edgeConfiguration.getCustomerIdLSB() != 0) { + UUID customerUUID = new UUID(edgeConfiguration.getCustomerIdMSB(), edgeConfiguration.getCustomerIdLSB()); + this.customerId = new CustomerId(customerUUID); + return !this.customerId.equals(previousCustomerId); + } else { + this.customerId = null; + return false; + } + } + + private EdgeId getEdgeId(EdgeConfiguration edgeConfiguration) { + UUID edgeUUID = new UUID(edgeConfiguration.getEdgeIdMSB(), edgeConfiguration.getEdgeIdLSB()); + return new EdgeId(edgeUUID); + } + + private void saveOrUpdateEdge(TenantId tenantId, EdgeConfiguration edgeConfiguration) throws ExecutionException, InterruptedException { + EdgeId edgeId = getEdgeId(edgeConfiguration); + edgeCloudProcessor.processEdgeConfigurationMsgFromCloud(tenantId, edgeConfiguration); + cloudEventService.saveCloudEvent(tenantId, CloudEventType.EDGE, EdgeEventActionType.ATTRIBUTES_REQUEST, edgeId, null); + cloudEventService.saveCloudEvent(tenantId, CloudEventType.EDGE, EdgeEventActionType.RELATION_REQUEST, edgeId, null); + } + + private EdgeSettings constructEdgeSettings(EdgeConfiguration edgeConfiguration) { + EdgeSettings edgeSettings = new EdgeSettings(); + UUID edgeUUID = new UUID(edgeConfiguration.getEdgeIdMSB(), edgeConfiguration.getEdgeIdLSB()); + edgeSettings.setEdgeId(edgeUUID.toString()); + UUID tenantUUID = new UUID(edgeConfiguration.getTenantIdMSB(), edgeConfiguration.getTenantIdLSB()); + edgeSettings.setTenantId(tenantUUID.toString()); + edgeSettings.setName(edgeConfiguration.getName()); + edgeSettings.setType(edgeConfiguration.getType()); + edgeSettings.setRoutingKey(edgeConfiguration.getRoutingKey()); + edgeSettings.setFullSyncRequired(true); + return edgeSettings; + } + + private void onDownlink(DownlinkMsg downlinkMsg) { + boolean edgeCustomerIdUpdated = updateCustomerIdIfRequired(downlinkMsg); + if (this.syncInProgress && downlinkMsg.hasSyncCompletedMsg()) { + log.trace("[{}] downlinkMsg hasSyncCompletedMsg = true", downlinkMsg); + this.syncInProgress = false; + } + ListenableFuture> future = + downlinkMessageService.processDownlinkMsg(tenantId, customerId, downlinkMsg, this.currentEdgeSettings); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List result) { + log.trace("[{}] DownlinkMsg has been processed successfully! DownlinkMsgId {}", routingKey, downlinkMsg.getDownlinkMsgId()); + DownlinkResponseMsg downlinkResponseMsg = DownlinkResponseMsg.newBuilder() + .setDownlinkMsgId(downlinkMsg.getDownlinkMsgId()) + .setSuccess(true).build(); + edgeRpcClient.sendDownlinkResponseMsg(downlinkResponseMsg); + if (downlinkMsg.hasEdgeConfiguration()) { + if (edgeCustomerIdUpdated && !syncInProgress) { + log.info("Edge customer id has been updated. Sending sync request..."); + edgeRpcClient.sendSyncRequestMsg(false); + syncInProgress = true; + } + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to process DownlinkMsg! DownlinkMsgId {}", routingKey, downlinkMsg.getDownlinkMsgId()); + String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); + DownlinkResponseMsg downlinkResponseMsg = DownlinkResponseMsg.newBuilder() + .setDownlinkMsgId(downlinkMsg.getDownlinkMsgId()) + .setSuccess(false).setErrorMsg(errorMsg).build(); + edgeRpcClient.sendDownlinkResponseMsg(downlinkResponseMsg); + } + }, MoreExecutors.directExecutor()); + } + + private boolean updateCustomerIdIfRequired(DownlinkMsg downlinkMsg) { + if (downlinkMsg.hasEdgeConfiguration()) { + return setOrUpdateCustomerId(downlinkMsg.getEdgeConfiguration()); + } else { + return false; + } + } + + private void updateConnectivityStatus(boolean activityState) { + if (tenantId != null) { + save(DefaultDeviceStateService.ACTIVITY_STATE, activityState); + if (activityState) { + save(DefaultDeviceStateService.LAST_CONNECT_TIME, System.currentTimeMillis()); + } else { + save(DefaultDeviceStateService.LAST_DISCONNECT_TIME, System.currentTimeMillis()); + } + } + } + + private void scheduleReconnect(Exception e) { + initialized = false; + + updateConnectivityStatus(false); + + if (sendUplinkFuture == null) { + sendUplinkFuture = uplinkExecutor.scheduleAtFixedRate(() -> { + log.info("Trying to reconnect due to the error: {}!", e.getMessage()); + try { + edgeRpcClient.disconnect(true); + } catch (Exception ex) { + log.error("Exception during disconnect: {}", ex.getMessage()); + } + try { + edgeRpcClient.connect(routingKey, routingSecret, + this::onUplinkResponse, + this::onEdgeUpdate, + this::onDownlink, + this::scheduleReconnect); + } catch (Exception ex) { + log.error("Exception during connect: {}", ex.getMessage()); + } + }, reconnectTimeoutMs, reconnectTimeoutMs, TimeUnit.MILLISECONDS); + } + } + + private void save(String key, long value) { + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(key, value)); + } + + private void save(String key, boolean value) { + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(key, value)); + } + + private record AttributeSaveCallback(String key, Object value) implements FutureCallback { + + @Override + public void onSuccess(Void result) { + log.trace("Successfully updated attribute [{}] with value [{}]", key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to update attribute [{}] with value [{}]", key, value, t); + } + + } + + private UplinkMsg convertEventToUplink(CloudEvent cloudEvent) { + log.trace("Converting cloud event [{}]", cloudEvent); + try { + return switch (cloudEvent.getAction()) { + case UPDATED, ADDED, DELETED, ALARM_ACK, ALARM_CLEAR, ALARM_DELETE, CREDENTIALS_UPDATED, + RELATION_ADD_OR_UPDATE, RELATION_DELETED, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, + ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> convertEntityEventToUplink(cloudEvent); + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> + telemetryProcessor.convertTelemetryEventToUplink(cloudEvent.getTenantId(), cloudEvent); + case ATTRIBUTES_REQUEST -> telemetryProcessor.convertAttributesRequestEventToUplink(cloudEvent); + case RELATION_REQUEST -> relationProcessor.convertRelationRequestEventToUplink(cloudEvent); + case RPC_CALL -> deviceProcessor.convertRpcCallEventToUplink(cloudEvent); + case WIDGET_BUNDLE_TYPES_REQUEST -> widgetBundleProcessor.convertWidgetBundleTypesRequestEventToUplink(cloudEvent); + case ENTITY_VIEW_REQUEST -> entityViewProcessor.convertEntityViewRequestEventToUplink(cloudEvent); + default -> { + log.warn("Unsupported action type [{}]", cloudEvent); + yield null; + } + }; + } catch (Exception e) { + log.error("Exception during converting events from queue, skipping event [{}]", cloudEvent, e); + return null; + } + } + + private UplinkMsg convertEntityEventToUplink(CloudEvent cloudEvent) { + log.trace("Executing convertEntityEventToUplink cloudEvent [{}], edgeEventAction [{}]", cloudEvent, cloudEvent.getAction()); + EdgeVersion edgeVersion = EdgeVersion.V_LATEST; + + return switch (cloudEvent.getType()) { + case DEVICE -> deviceProcessor.convertDeviceEventToUplink(cloudEvent.getTenantId(), cloudEvent, edgeVersion); + case DEVICE_PROFILE -> deviceProfileProcessor.convertDeviceProfileEventToUplink(cloudEvent, edgeVersion); + case ALARM -> alarmProcessor.convertAlarmEventToUplink(cloudEvent, edgeVersion); + case ALARM_COMMENT -> alarmProcessor.convertAlarmCommentEventToUplink(cloudEvent, edgeVersion); + case ASSET -> assetProcessor.convertAssetEventToUplink(cloudEvent, edgeVersion); + case ASSET_PROFILE -> assetProfileProcessor.convertAssetProfileEventToUplink(cloudEvent, edgeVersion); + case DASHBOARD -> dashboardProcessor.convertDashboardEventToUplink(cloudEvent, edgeVersion); + case ENTITY_VIEW -> entityViewProcessor.convertEntityViewEventToUplink(cloudEvent, edgeVersion); + case RELATION -> relationProcessor.convertRelationEventToUplink(cloudEvent, edgeVersion); + case TB_RESOURCE -> resourceCloudProcessor.convertResourceEventToUplink(cloudEvent, edgeVersion); + default -> { + log.warn("Unsupported cloud event type [{}]", cloudEvent); + yield null; + } + }; + } + + protected ListenableFuture processCloudEvents(List cloudEvents, boolean isGeneralMsg) { + interruptPreviousSendUplinkMsgsTask(); + sendUplinkFutureResult = SettableFuture.create(); + + log.trace("[{}] event(s) are going to be converted.", cloudEvents.size()); + List uplinkMsgPack = cloudEvents.stream() + .map(this::convertEventToUplink) + .filter(Objects::nonNull) + .toList(); + + if (uplinkMsgPack.isEmpty()) { + return Futures.immediateFuture(true); + } + + processMsgPack(uplinkMsgPack); + return sendUplinkFutureResult; + } + + private void interruptPreviousSendUplinkMsgsTask() { + if (sendUplinkFutureResult != null && !sendUplinkFutureResult.isDone()) { + log.debug("[{}] Stopping send uplink future now!", tenantId); + sendUplinkFutureResult.set(true); + } + if (sendUplinkFuture != null && !sendUplinkFuture.isCancelled()) { + sendUplinkFuture.cancel(true); + } + } + + private void processMsgPack(List uplinkMsgPack) { + pendingMsgMap.clear(); + uplinkMsgPack.forEach(msg -> pendingMsgMap.put(msg.getUplinkMsgId(), msg)); + LinkedBlockingQueue orderedPendingMsgQueue = new LinkedBlockingQueue<>(pendingMsgMap.values()); + sendUplinkFuture = uplinkExecutor.schedule(() -> { + int attempt = 1; + boolean success; + do { + log.trace("[{}] uplink msg(s) are going to be send.", pendingMsgMap.values().size()); + + success = sendUplinkMsgPack(orderedPendingMsgQueue) && pendingMsgMap.isEmpty(); + + if (!success) { + log.warn("Failed to deliver the batch: {}, attempt: {}", pendingMsgMap.values(), attempt); + try { + Thread.sleep(cloudEventStorageSettings.getSleepIntervalBetweenBatches()); + + if (isRateLimitViolated) { + isRateLimitViolated = false; + TimeUnit.SECONDS.sleep(60); + } + } catch (InterruptedException e) { + log.error("Error during sleep between batches or on rate limit violation", e); + } + } + + attempt++; + + if (attempt > MAX_SEND_UPLINK_ATTEMPTS) { + log.warn("Failed to deliver the batch: after {} attempts. Next messages are going to be discarded {}", + MAX_SEND_UPLINK_ATTEMPTS, pendingMsgMap.values()); + sendUplinkFutureResult.set(true); + return; + } + } while (!success); + sendUplinkFutureResult.set(false); + }, 0L, TimeUnit.MILLISECONDS); + } + + private boolean sendUplinkMsgPack(LinkedBlockingQueue orderedPendingMsgQueue) { + try { + boolean success; + + sendingInProgress = true; + latch = new CountDownLatch(pendingMsgMap.values().size()); + orderedPendingMsgQueue.forEach(this::sendUplinkMsg); + + success = latch.await(uplinkPackTimeoutSec, TimeUnit.SECONDS); + sendingInProgress = false; + + return success; + } catch (InterruptedException e) { + log.error("sendUplinkMsgPack throw InterruptedException", e); + throw new RuntimeException("Interrupted while waiting for latch. " + e); + } + } + + private void sendUplinkMsg(UplinkMsg uplinkMsg) { + if (edgeRpcClient.getServerMaxInboundMessageSize() == 0 || + uplinkMsg.getSerializedSize() <= edgeRpcClient.getServerMaxInboundMessageSize()) { + edgeRpcClient.sendUplinkMsg(uplinkMsg); + } else { + log.error("Uplink msg size [{}] exceeds server max inbound message size [{}]. Skipping this message. " + + "Please increase value of EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the server and restart it. Message {}", + uplinkMsg.getSerializedSize(), edgeRpcClient.getServerMaxInboundMessageSize(), uplinkMsg); + pendingMsgMap.remove(uplinkMsg.getUplinkMsgId()); + latch.countDown(); + } + } + + @FunctionalInterface + protected interface CloudEventFinder { + PageData find(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/BaseUplinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/BaseUplinkMessageService.java deleted file mode 100644 index 228414491d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cloud/BaseUplinkMessageService.java +++ /dev/null @@ -1,370 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cloud; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.edge.rpc.EdgeRpcClient; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cloud.CloudEvent; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cloud.CloudEventService; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.UplinkMsg; -import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; -import org.thingsboard.server.service.cloud.rpc.CloudEventStorageSettings; -import org.thingsboard.server.service.cloud.rpc.processor.AlarmCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.AssetCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.AssetProfileCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.DashboardCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.DeviceCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.DeviceProfileCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.EntityViewCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.RelationCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.ResourceCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.TelemetryCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.WidgetBundleCloudProcessor; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -@Slf4j -public abstract class BaseUplinkMessageService { - - protected static final String QUEUE_SEQ_ID_OFFSET_ATTR_KEY = "queueSeqIdOffset"; - private static final String RATE_LIMIT_REACHED = "Rate limit reached"; - private static final int MAX_UPLINK_ATTEMPTS = 10; // max number of attemps to send downlink message if edge connected - - private static final ReentrantLock uplinkMsgsPackLock = new ReentrantLock(); - private final ConcurrentMap pendingMsgsMap = new ConcurrentHashMap<>(); - private CountDownLatch latch; - - private volatile boolean isRateLimitViolated = false; - private volatile boolean sendingInProgress = false; - - @Value("${cloud.uplink_pack_timeout_sec:60}") - private long uplinkPackTimeoutSec; - - @Autowired - protected CloudEventService cloudEventService; - - @Autowired - private CloudEventStorageSettings cloudEventStorageSettings; - - @Autowired - private RelationCloudProcessor relationProcessor; - - @Autowired - private DeviceCloudProcessor deviceProcessor; - - @Autowired - private DeviceProfileCloudProcessor deviceProfileProcessor; - - @Autowired - private AlarmCloudProcessor alarmProcessor; - - @Autowired - private TelemetryCloudProcessor telemetryProcessor; - - @Autowired - private WidgetBundleCloudProcessor widgetBundleProcessor; - - @Autowired - private EntityViewCloudProcessor entityViewProcessor; - - @Autowired - private DashboardCloudProcessor dashboardProcessor; - - @Autowired - private AssetCloudProcessor assetProcessor; - - @Autowired - private AssetProfileCloudProcessor assetProfileProcessor; - - @Autowired - private ResourceCloudProcessor resourceCloudProcessor; - - @Autowired - private AttributesService attributesService; - - @Autowired - private EdgeRpcClient edgeRpcClient; - - @Autowired - private DbCallbackExecutorService dbCallbackExecutorService; - - public void processCloudEvents(TenantId tenantId, Long queueSeqIdStart, TimePageLink pageLink) throws Exception { - PageData cloudEvents; - boolean success; - do { - cloudEvents = findCloudEvents(tenantId, queueSeqIdStart, null, pageLink); - if (cloudEvents.getData().isEmpty()) { - log.info("seqId column of {} table started new cycle", getTableName()); - cloudEvents = findCloudEventsFromBeginning(tenantId, pageLink); - } - log.trace("[{}] event(s) are going to be converted.", cloudEvents.getData().size()); - List uplinkMsgsPack = convertToUplinkMsgsPack(tenantId, cloudEvents.getData()); - if (!uplinkMsgsPack.isEmpty()) { - success = sendUplinkMsgsPack(uplinkMsgsPack); - } else { - success = true; - } - if (success && cloudEvents.getTotalElements() > 0) { - CloudEvent latestCloudEvent = cloudEvents.getData().get(cloudEvents.getData().size() - 1); - try { - Long newStartTs = Uuids.unixTimestamp(latestCloudEvent.getUuidId()); - updateQueueStartTsSeqIdOffset(tenantId, newStartTs, latestCloudEvent.getSeqId()); - log.debug("Queue offset was updated [{}][{}][{}]", latestCloudEvent.getUuidId(), newStartTs, latestCloudEvent.getSeqId()); - } catch (Exception e) { - log.error("Failed to update queue offset [{}]", latestCloudEvent); - } - } - if (success) { - pageLink = pageLink.nextPageLink(); - } - if (newMessagesAvailableInGeneralQueue(tenantId)) { - return; - } - } while (!success || cloudEvents.hasNext()); - } - - protected abstract String getTableName(); - protected abstract boolean newMessagesAvailableInGeneralQueue(TenantId tenantId); - protected abstract void updateQueueStartTsSeqIdOffset(TenantId tenantId, Long newStartTs, Long newSeqId); - - public void processHandleMessages(TenantId tenantId) throws Exception { - Long cloudEventsQueueSeqIdStart = getQueueSeqIdStart(tenantId).get(); - TimePageLink cloudEventsPageLink = newCloudEventsAvailable(tenantId, cloudEventsQueueSeqIdStart); - if (cloudEventsPageLink != null) { - processCloudEvents(tenantId, cloudEventsQueueSeqIdStart, cloudEventsPageLink); - } - } - - protected abstract ListenableFuture getQueueStartTs(TenantId tenantId); - protected abstract ListenableFuture getQueueSeqIdStart(TenantId tenantId); - - protected ListenableFuture getLongAttrByKey(TenantId tenantId, String attrKey) { - ListenableFuture> future = - attributesService.find(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attrKey); - return Futures.transform(future, attributeKvEntryOpt -> { - if (attributeKvEntryOpt != null && attributeKvEntryOpt.isPresent()) { - AttributeKvEntry attributeKvEntry = attributeKvEntryOpt.get(); - return attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } else { - return 0L; - } - }, dbCallbackExecutorService); - } - - public TimePageLink newCloudEventsAvailable(TenantId tenantId, Long queueSeqIdStart) { - try { - long queueStartTs = getQueueStartTs(tenantId).get(); - long queueEndTs = queueStartTs > 0 ? queueStartTs + TimeUnit.DAYS.toMillis(1) : System.currentTimeMillis(); - TimePageLink pageLink = new TimePageLink(cloudEventStorageSettings.getMaxReadRecordsCount(), - 0, null, null, queueStartTs, queueEndTs); - PageData cloudEvents = findCloudEvents(tenantId, queueSeqIdStart, null, pageLink); - if (cloudEvents.getData().isEmpty()) { - // check if new cycle started (seq_id starts from '1') - cloudEvents = findCloudEventsFromBeginning(tenantId, pageLink); - if (cloudEvents.getData().stream().anyMatch(ce -> ce.getSeqId() == 1)) { - log.info("newCloudEventsAvailable: new cycle started (seq_id starts from '1')!"); - return pageLink; - } else { - while (queueEndTs < System.currentTimeMillis()) { - log.trace("newCloudEventsAvailable: queueEndTs < System.currentTimeMillis() [{}] [{}]", queueEndTs, System.currentTimeMillis()); - queueStartTs = queueEndTs; - queueEndTs = queueEndTs + TimeUnit.DAYS.toMillis(1); - pageLink = new TimePageLink(cloudEventStorageSettings.getMaxReadRecordsCount(), - 0, null, null, queueStartTs, queueEndTs); - cloudEvents = findCloudEvents(tenantId, queueSeqIdStart, null, pageLink); - if (!cloudEvents.getData().isEmpty()) { - return pageLink; - } - } - return null; - } - } else { - return pageLink; - } - } catch (Exception e) { - log.warn("Failed to check newCloudEventsAvailable!", e); - return null; - } - } - - protected PageData findCloudEventsFromBeginning(TenantId tenantId, TimePageLink pageLink) { - long seqIdEnd = Integer.toUnsignedLong(cloudEventStorageSettings.getMaxReadRecordsCount()); - seqIdEnd = Math.max(seqIdEnd, 50L); - return findCloudEvents(tenantId, 0L, seqIdEnd, pageLink); - } - - protected abstract PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); - - protected void updateQueueStartTsSeqIdOffset(TenantId tenantId, String attrStartTsKey, String attrSeqIdKey, Long startTs, Long seqIdOffset) { - log.trace("updateQueueStartTsSeqIdOffset [{}][{}][{}][{}]", attrStartTsKey, attrSeqIdKey, startTs, seqIdOffset); - List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(attrStartTsKey, startTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(attrSeqIdKey, seqIdOffset), System.currentTimeMillis())); - attributesService.save(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attributes); - } - - public void onUplinkResponse(UplinkResponseMsg msg) { - try { - if (sendingInProgress) { - if (msg.getSuccess()) { - pendingMsgsMap.remove(msg.getUplinkMsgId()); - log.debug("Msg has been processed successfully! {}", msg); - } else if (msg.getErrorMsg().contains(RATE_LIMIT_REACHED)) { - log.warn("Msg processing failed! {}", RATE_LIMIT_REACHED); - isRateLimitViolated = true; - } else { - log.error("Msg processing failed! Error msg: {}", msg.getErrorMsg()); - } - latch.countDown(); - } - } catch (Exception e) { - log.error("Can't process uplink response message [{}]", msg, e); - } - } - - protected boolean sendUplinkMsgsPack(List uplinkMsgsPack) throws InterruptedException { - uplinkMsgsPackLock.lock(); - try { - int attempt = 1; - boolean success; - LinkedBlockingQueue orderedPendingMsgsQueue = new LinkedBlockingQueue<>(); - pendingMsgsMap.clear(); - uplinkMsgsPack.forEach(msg -> { - pendingMsgsMap.put(msg.getUplinkMsgId(), msg); - orderedPendingMsgsQueue.add(msg); - }); - do { - log.trace("[{}] uplink msg(s) are going to be send.", pendingMsgsMap.values().size()); - sendingInProgress = true; - latch = new CountDownLatch(pendingMsgsMap.values().size()); - for (UplinkMsg uplinkMsg : orderedPendingMsgsQueue) { - if (edgeRpcClient.getServerMaxInboundMessageSize() != 0 && uplinkMsg.getSerializedSize() > edgeRpcClient.getServerMaxInboundMessageSize()) { - log.error("Uplink msg size [{}] exceeds server max inbound message size [{}]. Skipping this message. " + - "Please increase value of EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the server and restart it." + - "Message {}", - uplinkMsg.getSerializedSize(), edgeRpcClient.getServerMaxInboundMessageSize(), uplinkMsg); - pendingMsgsMap.remove(uplinkMsg.getUplinkMsgId()); - latch.countDown(); - } else { - edgeRpcClient.sendUplinkMsg(uplinkMsg); - } - } - success = latch.await(uplinkPackTimeoutSec, TimeUnit.SECONDS); - sendingInProgress = false; - success = success && pendingMsgsMap.isEmpty(); - if (!success) { - log.warn("Failed to deliver the batch: {}, attempt: {}", pendingMsgsMap.values(), attempt); - } - if (!success) { - try { - Thread.sleep(cloudEventStorageSettings.getSleepIntervalBetweenBatches()); - } catch (InterruptedException e) { - log.error("Error during sleep between batches", e); - } - } - if (!success && isRateLimitViolated) { - isRateLimitViolated = false; - try { - TimeUnit.SECONDS.sleep(60); - } catch (InterruptedException e) { - log.error("Error during sleep on rate limit violation", e); - } - } - attempt++; - if (attempt > MAX_UPLINK_ATTEMPTS) { - log.warn("Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", - MAX_UPLINK_ATTEMPTS, pendingMsgsMap.values()); - return true; - } - } while (!success); - return true; - } finally { - uplinkMsgsPackLock.unlock(); - } - } - - private List convertToUplinkMsgsPack(TenantId tenantId, List cloudEvents) { - List result = new ArrayList<>(); - for (CloudEvent cloudEvent : cloudEvents) { - log.trace("Converting cloud event [{}]", cloudEvent); - UplinkMsg uplinkMsg = null; - try { - switch (cloudEvent.getAction()) { - case UPDATED, ADDED, DELETED, ALARM_ACK, ALARM_CLEAR, ALARM_DELETE, CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> - uplinkMsg = convertEntityEventToUplink(tenantId, cloudEvent); - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> - uplinkMsg = telemetryProcessor.convertTelemetryEventToUplink(tenantId, cloudEvent); - case ATTRIBUTES_REQUEST -> uplinkMsg = telemetryProcessor.convertAttributesRequestEventToUplink(cloudEvent); - case RELATION_REQUEST -> uplinkMsg = relationProcessor.convertRelationRequestEventToUplink(cloudEvent); - case RPC_CALL -> uplinkMsg = deviceProcessor.convertRpcCallEventToUplink(cloudEvent); - case WIDGET_BUNDLE_TYPES_REQUEST -> uplinkMsg = widgetBundleProcessor.convertWidgetBundleTypesRequestEventToUplink(cloudEvent); - case ENTITY_VIEW_REQUEST -> uplinkMsg = entityViewProcessor.convertEntityViewRequestEventToUplink(cloudEvent); - } - } catch (Exception e) { - log.error("Exception during converting events from queue, skipping event [{}]", cloudEvent, e); - } - if (uplinkMsg != null) { - result.add(uplinkMsg); - } - } - return result; - } - - private UplinkMsg convertEntityEventToUplink(TenantId tenantId, CloudEvent cloudEvent) { - log.trace("Executing convertEntityEventToUplink, cloudEvent [{}], edgeEventAction [{}]", cloudEvent, cloudEvent.getAction()); - EdgeVersion edgeVersion = EdgeVersion.V_LATEST; - return switch (cloudEvent.getType()) { - case DEVICE -> deviceProcessor.convertDeviceEventToUplink(tenantId, cloudEvent, edgeVersion); - case DEVICE_PROFILE -> deviceProfileProcessor.convertDeviceProfileEventToUplink(cloudEvent, edgeVersion); - case ALARM -> alarmProcessor.convertAlarmEventToUplink(cloudEvent, edgeVersion); - case ALARM_COMMENT -> alarmProcessor.convertAlarmCommentEventToUplink(cloudEvent, edgeVersion); - case ASSET -> assetProcessor.convertAssetEventToUplink(cloudEvent, edgeVersion); - case ASSET_PROFILE -> assetProfileProcessor.convertAssetProfileEventToUplink(cloudEvent, edgeVersion); - case DASHBOARD -> dashboardProcessor.convertDashboardEventToUplink(cloudEvent, edgeVersion); - case ENTITY_VIEW -> entityViewProcessor.convertEntityViewEventToUplink(cloudEvent, edgeVersion); - case RELATION -> relationProcessor.convertRelationEventToUplink(cloudEvent, edgeVersion); - case TB_RESOURCE -> resourceCloudProcessor.convertResourceEventToUplink(cloudEvent, edgeVersion); - default -> { - log.warn("Unsupported cloud event type [{}]", cloudEvent); - yield null; - } - }; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/TsUplinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/CloudEventMigrationService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/cloud/TsUplinkMessageService.java rename to application/src/main/java/org/thingsboard/server/service/cloud/CloudEventMigrationService.java index 5376105e9e..3b150fe3f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/TsUplinkMessageService.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/CloudEventMigrationService.java @@ -15,5 +15,12 @@ */ package org.thingsboard.server.service.cloud; -public interface TsUplinkMessageService extends UplinkMessageService { +public interface CloudEventMigrationService { + + boolean isMigrated(); + + boolean isTsMigrated(); + + void migrateUnprocessedEventToKafka(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/CloudManagerService.java b/application/src/main/java/org/thingsboard/server/service/cloud/CloudManagerService.java deleted file mode 100644 index 1325c74196..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cloud/CloudManagerService.java +++ /dev/null @@ -1,454 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cloud; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.edge.rpc.EdgeRpcClient; -import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EdgeUtils; -import org.thingsboard.server.common.data.cloud.CloudEventType; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.edge.EdgeSettings; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cloud.CloudEventService; -import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; -import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; -import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; -import org.thingsboard.server.service.cloud.rpc.CloudEventStorageSettings; -import org.thingsboard.server.service.cloud.rpc.processor.CustomerCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.EdgeCloudProcessor; -import org.thingsboard.server.service.cloud.rpc.processor.TenantCloudProcessor; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; - -import javax.annotation.PreDestroy; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -@Service -@Slf4j -public class CloudManagerService { - - private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - private static final String QUEUE_SEQ_ID_OFFSET_ATTR_KEY = "queueSeqIdOffset"; - private static final String QUEUE_TS_KV_START_TS_ATTR_KEY = "queueTsKvStartTs"; - private static final String QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY = "queueTsKvSeqIdOffset"; - - - @Value("${cloud.routingKey}") - private String routingKey; - - @Value("${cloud.secret}") - private String routingSecret; - - @Value("${cloud.reconnect_timeout}") - private long reconnectTimeoutMs; - - @Autowired - private EdgeService edgeService; - - @Autowired - private AttributesService attributesService; - - @Autowired - protected TelemetrySubscriptionService tsSubService; - - @Autowired - protected TbClusterService tbClusterService; - - @Autowired - private CloudEventStorageSettings cloudEventStorageSettings; - - @Autowired - private GeneralUplinkMessageService generalUplinkMessageService; - - @Autowired - private TsUplinkMessageService tsUplinkMessageService; - - @Autowired - private DownlinkMessageService downlinkMessageService; - - @Autowired - private EdgeRpcClient edgeRpcClient; - - @Autowired - private EdgeCloudProcessor edgeCloudProcessor; - - - @Autowired - private TenantCloudProcessor tenantProcessor; - - @Autowired - private CustomerCloudProcessor customerProcessor; - - - - @Autowired - private CloudEventService cloudEventService; - - @Autowired - private ConfigurableApplicationContext context; - - - - private EdgeSettings currentEdgeSettings; - - private Long queueStartTs; - - - private ExecutorService executor; - private ScheduledExecutorService reconnectScheduler; - private ScheduledFuture scheduledFuture; - private ScheduledExecutorService shutdownExecutor; - - private volatile boolean initialized; - private volatile boolean syncInProgress = false; - - - private TenantId tenantId; - private CustomerId customerId; - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - if (validateRoutingKeyAndSecret()) { - log.info("Starting Cloud Edge service"); - edgeRpcClient.connect(routingKey, routingSecret, - this::onUplinkResponse, - this::onEdgeUpdate, - this::onDownlink, - this::scheduleReconnect); - executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("cloud-manager")); - reconnectScheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("cloud-manager-reconnect")); - processHandleMessages(); - } - } - - private boolean validateRoutingKeyAndSecret() { - if (StringUtils.isBlank(routingKey) || StringUtils.isBlank(routingSecret)) { - shutdownExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("cloud-manager-shutdown")); - shutdownExecutor.scheduleAtFixedRate(() -> log.error( - "Routing Key and Routing Secret must be provided! " + - "Please configure Routing Key and Routing Secret in the tb-edge.yml file " + - "or add CLOUD_ROUTING_KEY and CLOUD_ROUTING_SECRET variable to the tb-edge.conf file. " + - "ThingsBoard Edge is not going to connect to cloud!"), 0, 10, TimeUnit.SECONDS); - return false; - } - return true; - } - - @PreDestroy - public void destroy() throws InterruptedException { - if (shutdownExecutor != null) { - shutdownExecutor.shutdown(); - } - - updateConnectivityStatus(false); - - String edgeId = currentEdgeSettings != null ? currentEdgeSettings.getEdgeId() : ""; - log.info("[{}] Starting destroying process", edgeId); - try { - edgeRpcClient.disconnect(false); - } catch (Exception e) { - log.error("Exception during disconnect", e); - } - if (executor != null) { - executor.shutdownNow(); - } - if (reconnectScheduler != null) { - reconnectScheduler.shutdownNow(); - } - log.info("[{}] Destroy was successful", edgeId); - } - - private void processHandleMessages() { - executor.submit(() -> { - while (!Thread.interrupted()) { - try { - if (initialized) { - generalUplinkMessageService.processHandleMessages(tenantId); - tsUplinkMessageService.processHandleMessages(tenantId); - try { - Thread.sleep(cloudEventStorageSettings.getNoRecordsSleepInterval()); - } catch (InterruptedException e) { - log.error("Error during sleep", e); - } - } else { - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - } catch (Exception e) { - log.warn("Failed to process messages handling!", e); - } - } - }); - } - - private void updateQueueStartTsSeqIdOffset(String attrStartTsKey, String attrSeqIdKey, Long startTs, Long seqIdOffset) { - log.trace("updateQueueStartTsSeqIdOffset [{}][{}]", startTs, seqIdOffset); - List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(attrStartTsKey, startTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(attrSeqIdKey, seqIdOffset), System.currentTimeMillis())); - attributesService.save(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attributes); - } - - private void onUplinkResponse(UplinkResponseMsg msg) { - generalUplinkMessageService.onUplinkResponse(msg); - tsUplinkMessageService.onUplinkResponse(msg); - } - - private void onEdgeUpdate(EdgeConfiguration edgeConfiguration) { - try { - if (scheduledFuture != null) { - scheduledFuture.cancel(true); - scheduledFuture = null; - } - - if ("CE".equals(edgeConfiguration.getCloudType())) { - initAndUpdateEdgeSettings(edgeConfiguration); - } else { - new Thread(() -> { - log.error("Terminating application. CE edge can be connected only to CE server version..."); - int exitCode = -1; - int appExitCode = exitCode; - try { - appExitCode = SpringApplication.exit(context, () -> exitCode); - } finally { - System.exit(appExitCode); - } - }, "Shutdown Thread").start(); - } - } catch (Exception e) { - log.error("Can't process edge configuration message [{}]", edgeConfiguration, e); - } - } - - private void initAndUpdateEdgeSettings(EdgeConfiguration edgeConfiguration) throws Exception { - this.tenantId = new TenantId(new UUID(edgeConfiguration.getTenantIdMSB(), edgeConfiguration.getTenantIdLSB())); - - this.currentEdgeSettings = cloudEventService.findEdgeSettings(this.tenantId); - EdgeSettings newEdgeSettings = constructEdgeSettings(edgeConfiguration); - if (this.currentEdgeSettings == null || !this.currentEdgeSettings.getEdgeId().equals(newEdgeSettings.getEdgeId())) { - tenantProcessor.cleanUp(); - this.currentEdgeSettings = newEdgeSettings; - updateQueueStartTsSeqIdOffset(QUEUE_START_TS_ATTR_KEY, QUEUE_SEQ_ID_OFFSET_ATTR_KEY, System.currentTimeMillis(), 0L); - updateQueueStartTsSeqIdOffset(QUEUE_TS_KV_START_TS_ATTR_KEY, QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY, System.currentTimeMillis(), 0L); - } else { - log.trace("Using edge settings from DB {}", this.currentEdgeSettings); - } - - queueStartTs = generalUplinkMessageService.getQueueStartTs(tenantId).get(); - tenantProcessor.createTenantIfNotExists(this.tenantId, queueStartTs); - boolean edgeCustomerIdUpdated = setOrUpdateCustomerId(edgeConfiguration); - if (edgeCustomerIdUpdated) { - customerProcessor.createCustomerIfNotExists(this.tenantId, edgeConfiguration); - } - // TODO: voba - should sync be executed in some other cases ??? - log.trace("Sending sync request, fullSyncRequired {}", this.currentEdgeSettings.isFullSyncRequired()); - edgeRpcClient.sendSyncRequestMsg(this.currentEdgeSettings.isFullSyncRequired()); - this.syncInProgress = true; - - cloudEventService.saveEdgeSettings(tenantId, this.currentEdgeSettings); - - saveOrUpdateEdge(tenantId, edgeConfiguration); - - updateConnectivityStatus(true); - - initialized = true; - } - - private boolean setOrUpdateCustomerId(EdgeConfiguration edgeConfiguration) { - EdgeId edgeId = getEdgeId(edgeConfiguration); - Edge edge = edgeService.findEdgeById(tenantId, edgeId); - CustomerId previousCustomerId = null; - if (edge != null) { - previousCustomerId = edge.getCustomerId(); - } - if (edgeConfiguration.getCustomerIdMSB() != 0 && edgeConfiguration.getCustomerIdLSB() != 0) { - UUID customerUUID = new UUID(edgeConfiguration.getCustomerIdMSB(), edgeConfiguration.getCustomerIdLSB()); - this.customerId = new CustomerId(customerUUID); - return !this.customerId.equals(previousCustomerId); - } else { - this.customerId = null; - return false; - } - } - - private EdgeId getEdgeId(EdgeConfiguration edgeConfiguration) { - UUID edgeUUID = new UUID(edgeConfiguration.getEdgeIdMSB(), edgeConfiguration.getEdgeIdLSB()); - return new EdgeId(edgeUUID); - } - - private void saveOrUpdateEdge(TenantId tenantId, EdgeConfiguration edgeConfiguration) throws ExecutionException, InterruptedException { - EdgeId edgeId = getEdgeId(edgeConfiguration); - edgeCloudProcessor.processEdgeConfigurationMsgFromCloud(tenantId, edgeConfiguration); - cloudEventService.saveCloudEvent(tenantId, CloudEventType.EDGE, EdgeEventActionType.ATTRIBUTES_REQUEST, edgeId, null, queueStartTs); - cloudEventService.saveCloudEvent(tenantId, CloudEventType.EDGE, EdgeEventActionType.RELATION_REQUEST, edgeId, null, queueStartTs); - } - - private EdgeSettings constructEdgeSettings(EdgeConfiguration edgeConfiguration) { - EdgeSettings edgeSettings = new EdgeSettings(); - UUID edgeUUID = new UUID(edgeConfiguration.getEdgeIdMSB(), edgeConfiguration.getEdgeIdLSB()); - edgeSettings.setEdgeId(edgeUUID.toString()); - UUID tenantUUID = new UUID(edgeConfiguration.getTenantIdMSB(), edgeConfiguration.getTenantIdLSB()); - edgeSettings.setTenantId(tenantUUID.toString()); - edgeSettings.setName(edgeConfiguration.getName()); - edgeSettings.setType(edgeConfiguration.getType()); - edgeSettings.setRoutingKey(edgeConfiguration.getRoutingKey()); - edgeSettings.setFullSyncRequired(true); - return edgeSettings; - } - - private void onDownlink(DownlinkMsg downlinkMsg) { - boolean edgeCustomerIdUpdated = updateCustomerIdIfRequired(downlinkMsg); - if (this.syncInProgress && downlinkMsg.hasSyncCompletedMsg()) { - log.trace("[{}] downlinkMsg hasSyncCompletedMsg = true", downlinkMsg); - this.syncInProgress = false; - } - ListenableFuture> future = - downlinkMessageService.processDownlinkMsg(tenantId, customerId, downlinkMsg, this.currentEdgeSettings, queueStartTs); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List result) { - log.trace("[{}] DownlinkMsg has been processed successfully! DownlinkMsgId {}", routingKey, downlinkMsg.getDownlinkMsgId()); - DownlinkResponseMsg downlinkResponseMsg = DownlinkResponseMsg.newBuilder() - .setDownlinkMsgId(downlinkMsg.getDownlinkMsgId()) - .setSuccess(true).build(); - edgeRpcClient.sendDownlinkResponseMsg(downlinkResponseMsg); - if (downlinkMsg.hasEdgeConfiguration()) { - if (edgeCustomerIdUpdated && !syncInProgress) { - log.info("Edge customer id has been updated. Sending sync request..."); - edgeRpcClient.sendSyncRequestMsg(false); - syncInProgress = true; - } - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}] Failed to process DownlinkMsg! DownlinkMsgId {}", routingKey, downlinkMsg.getDownlinkMsgId()); - String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); - DownlinkResponseMsg downlinkResponseMsg = DownlinkResponseMsg.newBuilder() - .setDownlinkMsgId(downlinkMsg.getDownlinkMsgId()) - .setSuccess(false).setErrorMsg(errorMsg).build(); - edgeRpcClient.sendDownlinkResponseMsg(downlinkResponseMsg); - } - }, MoreExecutors.directExecutor()); - } - - private boolean updateCustomerIdIfRequired(DownlinkMsg downlinkMsg) { - if (downlinkMsg.hasEdgeConfiguration()) { - return setOrUpdateCustomerId(downlinkMsg.getEdgeConfiguration()); - } else { - return false; - } - } - - private void updateConnectivityStatus(boolean activityState) { - if (tenantId != null) { - save(DefaultDeviceStateService.ACTIVITY_STATE, activityState); - if (activityState) { - save(DefaultDeviceStateService.LAST_CONNECT_TIME, System.currentTimeMillis()); - } else { - save(DefaultDeviceStateService.LAST_DISCONNECT_TIME, System.currentTimeMillis()); - } - } - } - - private void scheduleReconnect(Exception e) { - initialized = false; - - updateConnectivityStatus(false); - - if (scheduledFuture == null) { - scheduledFuture = reconnectScheduler.scheduleAtFixedRate(() -> { - log.info("Trying to reconnect due to the error: {}!", e.getMessage()); - try { - edgeRpcClient.disconnect(true); - } catch (Exception ex) { - log.error("Exception during disconnect: {}", ex.getMessage()); - } - try { - edgeRpcClient.connect(routingKey, routingSecret, - this::onUplinkResponse, - this::onEdgeUpdate, - this::onDownlink, - this::scheduleReconnect); - } catch (Exception ex) { - log.error("Exception during connect: {}", ex.getMessage()); - } - }, reconnectTimeoutMs, reconnectTimeoutMs, TimeUnit.MILLISECONDS); - } - } - - private void save(String key, long value) { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(key, value)); - } - - private void save(String key, boolean value) { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(key, value)); - } - - private static class AttributeSaveCallback implements FutureCallback { - private final String key; - private final Object value; - - AttributeSaveCallback(String key, Object value) { - this.key = key; - this.value = value; - } - - @Override - public void onSuccess(@javax.annotation.Nullable Void result) { - log.trace("Successfully updated attribute [{}] with value [{}]", key, value); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to update attribute [{}] with value [{}]", key, value, t); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultCloudNotificationService.java b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultCloudNotificationService.java index f8c0ff7a8d..4eea45addd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultCloudNotificationService.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultCloudNotificationService.java @@ -36,8 +36,8 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.cloud.CloudEventService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.dao.cloud.CloudEventService; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; @@ -117,7 +117,7 @@ private ListenableFuture processEntity(TenantId tenantId, TransportProtos. EntityId entityId = EntityIdFactory.getByCloudEventTypeAndUuid(cloudEventType, new UUID(cloudNotificationMsg.getEntityIdMSB(), cloudNotificationMsg.getEntityIdLSB())); return switch (cloudEventActionType) { case ADDED, UPDATED, CREDENTIALS_UPDATED, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, DELETED -> - cloudEventService.saveCloudEventAsync(tenantId, cloudEventType, cloudEventActionType, entityId, null, 0L); + cloudEventService.saveCloudEventAsync(tenantId, cloudEventType, cloudEventActionType, entityId, null); default -> Futures.immediateFuture(null); }; } @@ -127,14 +127,14 @@ private ListenableFuture processAlarm(TenantId tenantId, TransportProtos.C AlarmId alarmId = new AlarmId(new UUID(cloudNotificationMsg.getEntityIdMSB(), cloudNotificationMsg.getEntityIdLSB())); if (EdgeEventActionType.DELETED.equals(actionType) || EdgeEventActionType.ALARM_DELETE.equals(actionType)) { Alarm deletedAlarm = JacksonUtil.fromString(cloudNotificationMsg.getEntityBody(), Alarm.class); - return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM, actionType, alarmId, JacksonUtil.valueToTree(deletedAlarm), 0L); + return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM, actionType, alarmId, JacksonUtil.valueToTree(deletedAlarm)); } ListenableFuture future = alarmService.findAlarmByIdAsync(tenantId, alarmId); return Futures.transformAsync(future, alarm -> { if (alarm != null) { CloudEventType cloudEventType = CloudUtils.getCloudEventTypeByEntityType(alarm.getOriginator().getEntityType()); if (cloudEventType != null) { - return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM, EdgeEventActionType.valueOf(cloudNotificationMsg.getCloudEventAction()), alarmId, null, 0L); + return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM, EdgeEventActionType.valueOf(cloudNotificationMsg.getCloudEventAction()), alarmId, null); } } return Futures.immediateFuture(null); @@ -148,12 +148,12 @@ public ListenableFuture processAlarmComment(TenantId tenantId, TransportPr if (alarmComment == null) { return Futures.immediateFuture(null); } - return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM_COMMENT, actionType, alarmId, JacksonUtil.valueToTree(alarmComment), 0L); + return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ALARM_COMMENT, actionType, alarmId, JacksonUtil.valueToTree(alarmComment)); } private ListenableFuture processRelation(TenantId tenantId, TransportProtos.CloudNotificationMsgProto cloudNotificationMsg) { EntityRelation relation = JacksonUtil.fromString(cloudNotificationMsg.getEntityBody(), EntityRelation.class); - return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.RELATION, EdgeEventActionType.valueOf(cloudNotificationMsg.getCloudEventAction()), null, JacksonUtil.valueToTree(relation), 0L); + return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.RELATION, EdgeEventActionType.valueOf(cloudNotificationMsg.getCloudEventAction()), null, JacksonUtil.valueToTree(relation)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultDownlinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultDownlinkMessageService.java index b2170611b9..6b99155732 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultDownlinkMessageService.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultDownlinkMessageService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.EdgeSettingsService; import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; @@ -60,6 +60,7 @@ import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; +import org.thingsboard.server.dao.cloud.CloudEventService; import org.thingsboard.server.service.cloud.rpc.processor.AdminSettingsCloudProcessor; import org.thingsboard.server.service.cloud.rpc.processor.AlarmCloudProcessor; import org.thingsboard.server.service.cloud.rpc.processor.AssetCloudProcessor; @@ -100,6 +101,9 @@ public class DefaultDownlinkMessageService implements DownlinkMessageService { @Autowired private CloudEventService cloudEventService; + @Autowired + private EdgeSettingsService edgeSettingsService; + @Autowired private EdgeCloudProcessor edgeCloudProcessor; @@ -175,17 +179,14 @@ public class DefaultDownlinkMessageService implements DownlinkMessageService { public ListenableFuture> processDownlinkMsg(TenantId tenantId, CustomerId edgeCustomerId, DownlinkMsg downlinkMsg, - EdgeSettings currentEdgeSettings, - Long queueStartTs) { + EdgeSettings currentEdgeSettings) { List> result = new ArrayList<>(); try { log.debug("[{}] Starting process DownlinkMsg. edgeCustomerId [{}], downlinkMsgId [{}],", tenantId, edgeCustomerId, downlinkMsg.getDownlinkMsgId()); - if (downlinkMsg.getWidgetTypeUpdateMsgCount() == 0) { - log.trace("DownlinkMsg Body {}", downlinkMsg); - } + logDownlinkMsg(downlinkMsg); if (downlinkMsg.hasSyncCompletedMsg()) { - result.add(updateSyncRequiredState(tenantId, edgeCustomerId, currentEdgeSettings, queueStartTs)); + result.add(updateSyncRequiredState(tenantId, edgeCustomerId, currentEdgeSettings)); } if (downlinkMsg.hasEdgeConfiguration()) { result.add(edgeCloudProcessor.processEdgeConfigurationMsgFromCloud(tenantId, downlinkMsg.getEdgeConfiguration())); @@ -212,7 +213,7 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, } if (downlinkMsg.getDeviceUpdateMsgCount() > 0) { for (DeviceUpdateMsg deviceUpdateMsg : downlinkMsg.getDeviceUpdateMsgList()) { - result.add(deviceProcessor.processDeviceMsgFromCloud(tenantId, deviceUpdateMsg, queueStartTs)); + result.add(deviceProcessor.processDeviceMsgFromCloud(tenantId, deviceUpdateMsg)); } } if (downlinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { @@ -221,18 +222,18 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, } } if (downlinkMsg.getAssetProfileUpdateMsgCount() > 0) { - for (AssetProfileUpdateMsg assetProfileUpdateMsg : downlinkMsg.getAssetProfileUpdateMsgList()) { + for (AssetProfileUpdateMsg assetProfileUpdateMsg : downlinkMsg.getAssetProfileUpdateMsgList()) { result.add(assetProfileProcessor.processAssetProfileMsgFromCloud(tenantId, assetProfileUpdateMsg)); } } if (downlinkMsg.getAssetUpdateMsgCount() > 0) { for (AssetUpdateMsg assetUpdateMsg : downlinkMsg.getAssetUpdateMsgList()) { - result.add(assetProcessor.processAssetMsgFromCloud(tenantId, assetUpdateMsg, queueStartTs)); + result.add(assetProcessor.processAssetMsgFromCloud(tenantId, assetUpdateMsg)); } } if (downlinkMsg.getEntityViewUpdateMsgCount() > 0) { for (EntityViewUpdateMsg entityViewUpdateMsg : downlinkMsg.getEntityViewUpdateMsgList()) { - result.add(entityViewProcessor.processEntityViewMsgFromCloud(tenantId, entityViewUpdateMsg, queueStartTs)); + result.add(entityViewProcessor.processEntityViewMsgFromCloud(tenantId, entityViewUpdateMsg)); } } if (downlinkMsg.getRuleChainUpdateMsgCount() > 0) { @@ -247,7 +248,7 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, } if (downlinkMsg.getDashboardUpdateMsgCount() > 0) { for (DashboardUpdateMsg dashboardUpdateMsg : downlinkMsg.getDashboardUpdateMsgList()) { - result.add(dashboardProcessor.processDashboardMsgFromCloud(tenantId, dashboardUpdateMsg, edgeCustomerId, queueStartTs)); + result.add(dashboardProcessor.processDashboardMsgFromCloud(tenantId, dashboardUpdateMsg, edgeCustomerId)); } } if (downlinkMsg.getAlarmUpdateMsgCount() > 0) { @@ -264,7 +265,7 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, for (CustomerUpdateMsg customerUpdateMsg : downlinkMsg.getCustomerUpdateMsgList()) { sequenceDependencyLock.lock(); try { - result.add(customerProcessor.processCustomerMsgFromCloud(tenantId, customerUpdateMsg, queueStartTs)); + result.add(customerProcessor.processCustomerMsgFromCloud(tenantId, customerUpdateMsg)); } finally { sequenceDependencyLock.unlock(); } @@ -289,7 +290,7 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, for (UserUpdateMsg userUpdateMsg : downlinkMsg.getUserUpdateMsgList()) { sequenceDependencyLock.lock(); try { - result.add(userProcessor.processUserMsgFromCloud(tenantId, userUpdateMsg, queueStartTs)); + result.add(userProcessor.processUserMsgFromCloud(tenantId, userUpdateMsg)); } finally { sequenceDependencyLock.unlock(); } @@ -363,19 +364,29 @@ public ListenableFuture> processDownlinkMsg(TenantId tenantId, return Futures.allAsList(result); } - private ListenableFuture updateSyncRequiredState(TenantId tenantId, CustomerId customerId, EdgeSettings currentEdgeSettings, Long queueStartTs) { + private void logDownlinkMsg(DownlinkMsg downlinkMsg) { + String msgStr = downlinkMsg != null ? downlinkMsg.toString() : "null"; + if (msgStr.length() > 10000) { + String truncatedMsg = msgStr.substring(0, 10000) + "... TRUNCATED"; + log.trace("DownlinkMsg Body (size: {}) {}", msgStr.length(), truncatedMsg); + } else { + log.trace("DownlinkMsg Body {}", msgStr); + } + } + + private ListenableFuture updateSyncRequiredState(TenantId tenantId, CustomerId customerId, EdgeSettings currentEdgeSettings) { log.debug("Marking full sync required to false"); if (currentEdgeSettings != null) { currentEdgeSettings.setFullSyncRequired(false); try { - cloudEventService.saveCloudEvent(tenantId, CloudEventType.TENANT, EdgeEventActionType.ATTRIBUTES_REQUEST, tenantId, null, queueStartTs); + cloudEventService.saveCloudEvent(tenantId, CloudEventType.TENANT, EdgeEventActionType.ATTRIBUTES_REQUEST, tenantId, null); if (customerId != null && !EntityId.NULL_UUID.equals(customerId.getId())) { - cloudEventService.saveCloudEvent(tenantId, CloudEventType.CUSTOMER, EdgeEventActionType.ATTRIBUTES_REQUEST, customerId, null, queueStartTs); + cloudEventService.saveCloudEvent(tenantId, CloudEventType.CUSTOMER, EdgeEventActionType.ATTRIBUTES_REQUEST, customerId, null); } } catch (Exception e) { log.error("Failed to request attributes for tenant and customer entities", e); } - return Futures.transform(cloudEventService.saveEdgeSettings(tenantId, currentEdgeSettings), + return Futures.transform(edgeSettingsService.saveEdgeSettings(tenantId, currentEdgeSettings), result -> { log.debug("Full sync required marked as false"); return null; @@ -389,7 +400,7 @@ private ListenableFuture updateSyncRequiredState(TenantId tenantId, Custom private ListenableFuture processDeviceCredentialsRequestMsg(TenantId tenantId, DeviceCredentialsRequestMsg deviceCredentialsRequestMsg) { if (deviceCredentialsRequestMsg.getDeviceIdMSB() != 0 && deviceCredentialsRequestMsg.getDeviceIdLSB() != 0) { DeviceId deviceId = new DeviceId(new UUID(deviceCredentialsRequestMsg.getDeviceIdMSB(), deviceCredentialsRequestMsg.getDeviceIdLSB())); - return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.DEVICE, EdgeEventActionType.CREDENTIALS_UPDATED, deviceId, null, 0L); + return cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.DEVICE, EdgeEventActionType.CREDENTIALS_UPDATED, deviceId, null); } else { return Futures.immediateFuture(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultGeneralUplinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultGeneralUplinkMessageService.java deleted file mode 100644 index a611d4eb31..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultGeneralUplinkMessageService.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cloud; - -import com.google.common.util.concurrent.ListenableFuture; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.cloud.CloudEvent; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; - -@Slf4j -@Service -public class DefaultGeneralUplinkMessageService extends BaseUplinkMessageService implements GeneralUplinkMessageService { - - private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - - @Override - protected PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return cloudEventService.findCloudEvents(tenantId, seqIdStart, seqIdEnd, pageLink); - } - - @Override - protected String getTableName() { - return "cloud_event"; - } - - @Override - protected boolean newMessagesAvailableInGeneralQueue(TenantId tenantId) { - return false; - } - - @Override - protected void updateQueueStartTsSeqIdOffset(TenantId tenantId, Long newStartTs, Long newSeqId) { - updateQueueStartTsSeqIdOffset(tenantId, QUEUE_START_TS_ATTR_KEY, QUEUE_SEQ_ID_OFFSET_ATTR_KEY, newStartTs, newSeqId); - } - - @Override - public ListenableFuture getQueueStartTs(TenantId tenantId) { - return getLongAttrByKey(tenantId, QUEUE_START_TS_ATTR_KEY); - } - - @Override - protected ListenableFuture getQueueSeqIdStart(TenantId tenantId) { - return getLongAttrByKey(tenantId, QUEUE_SEQ_ID_OFFSET_ATTR_KEY); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultTsUplinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/DefaultTsUplinkMessageService.java deleted file mode 100644 index d7e2b1971b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cloud/DefaultTsUplinkMessageService.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cloud; - -import com.google.common.util.concurrent.ListenableFuture; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.cloud.CloudEvent; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; - -@Slf4j -@Service -public class DefaultTsUplinkMessageService extends BaseUplinkMessageService implements TsUplinkMessageService { - - private static final String QUEUE_TS_KV_START_TS_ATTR_KEY = "queueTsKvStartTs"; - private static final String QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY = "queueTsKvSeqIdOffset"; - - @Autowired - private GeneralUplinkMessageService generalUplinkMessageService; - - @Override - protected PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return cloudEventService.findTsKvCloudEvents(tenantId, seqIdStart, seqIdEnd, pageLink); - } - - @Override - protected String getTableName() { - return "ts_kv_cloud_event"; - } - - @Override - protected boolean newMessagesAvailableInGeneralQueue(TenantId tenantId) { - try { - Long cloudEventsQueueSeqIdStart = getLongAttrByKey(tenantId, QUEUE_SEQ_ID_OFFSET_ATTR_KEY).get(); - TimePageLink cloudEventsPageLink = generalUplinkMessageService.newCloudEventsAvailable(tenantId, cloudEventsQueueSeqIdStart); - return cloudEventsPageLink != null; - } catch (Exception e) { - return false; - } - } - - @Override - protected void updateQueueStartTsSeqIdOffset(TenantId tenantId, Long newStartTs, Long newSeqId) { - updateQueueStartTsSeqIdOffset(tenantId, QUEUE_TS_KV_START_TS_ATTR_KEY, QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY, newStartTs, newSeqId); - } - - @Override - protected ListenableFuture getQueueStartTs(TenantId tenantId) { - return getLongAttrByKey(tenantId, QUEUE_TS_KV_START_TS_ATTR_KEY); - } - - @Override - protected ListenableFuture getQueueSeqIdStart(TenantId tenantId) { - return getLongAttrByKey(tenantId, QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/DownlinkMessageService.java b/application/src/main/java/org/thingsboard/server/service/cloud/DownlinkMessageService.java index a8664a8db0..8a36efdccf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/DownlinkMessageService.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/DownlinkMessageService.java @@ -28,7 +28,6 @@ public interface DownlinkMessageService { ListenableFuture> processDownlinkMsg(TenantId tenantId, CustomerId edgeCustomerId, DownlinkMsg downlinkMsg, - EdgeSettings currentEdgeSettings, - Long queueStartTs); + EdgeSettings currentEdgeSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventMigrationService.java b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventMigrationService.java new file mode 100644 index 0000000000..003ab643e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventMigrationService.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cloud; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.PostConstruct; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.cloud.CloudEvent; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.cloud.CloudEventService; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCloudEventMigrationService extends BaseCloudManagerService implements CloudEventMigrationService { + + @Autowired + private CloudEventService kafkaEventService; + + @Autowired + @Qualifier("postgresCloudEventService") + private CloudEventService postgresCloudEventService; + + @Getter + private volatile boolean isMigrated = false; + + @Getter + private volatile boolean isTsMigrated = false; + + private ExecutorService executor; + + @PostConstruct + private void onInit() { + executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("postgres-cloud-migrator")); + } + + @Override + public void migrateUnprocessedEventToKafka() { + executor.submit(() -> { + try { + while (!Thread.interrupted()) { + if (!initialized) { + TimeUnit.SECONDS.sleep(1); + continue; + } + + processMigration(); + + if (isMigrated && isTsMigrated) { + Thread.currentThread().interrupt(); + break; + } + } + } catch (Exception e) { + log.warn("Failed to process messages handling!", e); + } finally { + if (executor != null && !executor.isShutdown()) { + executor.shutdown(); + } + } + }); + } + + private void processMigration() throws Exception { + if (!isMigrated) { + CloudEventFinder finder = (tenantId, seqIdStart, seqIdEnd, link) -> postgresCloudEventService.findCloudEvents(tenantId, seqIdStart, seqIdEnd, link); + isMigrated = launchCloudEventProcessing(QUEUE_SEQ_ID_OFFSET_ATTR_KEY, QUEUE_START_TS_ATTR_KEY, true, finder); + } + + if (!isTsMigrated) { + CloudEventFinder finder = (tenantId, seqIdStart, seqIdEnd, link) -> postgresCloudEventService.findTsKvCloudEvents(tenantId, seqIdStart, seqIdEnd, link); + isTsMigrated = launchCloudEventProcessing(QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY, QUEUE_TS_KV_START_TS_ATTR_KEY, false, finder); + } + } + + private boolean launchCloudEventProcessing(String seqIdKey, String startTsKey, boolean isGeneralMsg, CloudEventFinder finder) throws Exception { + Long queueSeqIdStart = getLongAttrByKey(tenantId, seqIdKey).get(); + TimePageLink pageLink = newCloudEventsAvailable(tenantId, queueSeqIdStart, startTsKey, finder); + + if (pageLink != null) { + processUplinkMessages(pageLink, queueSeqIdStart, startTsKey, seqIdKey, isGeneralMsg, finder); + return false; + } + + return true; + } + + @Override + protected ListenableFuture processCloudEvents(List cloudEvents, boolean isGeneralMsg) { + for (CloudEvent cloudEvent : cloudEvents) { + if (isGeneralMsg) { + kafkaEventService.saveAsync(cloudEvent); + } else { + kafkaEventService.saveTsKvAsync(cloudEvent); + } + } + return Futures.immediateFuture(Boolean.FALSE); + } + + @Override + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) {} + + @Override + protected void launchUplinkProcessing() {} + + @PreDestroy + private void onDestroy() { + if (executor != null) { + executor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventService.java b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventService.java new file mode 100644 index 0000000000..4573823ac5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudEventService.java @@ -0,0 +1,128 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cloud; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.cloud.CloudEvent; +import org.thingsboard.server.common.data.cloud.CloudEventType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCloudEventProvider; + +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; + +@Slf4j +@Service +@Primary +@AllArgsConstructor +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCloudEventService implements CloudEventService { + + private final TbCloudEventProvider tbCloudEventProvider; + private final DataValidator cloudEventValidator; + + @Override + public void saveCloudEvent(TenantId tenantId, CloudEventType cloudEventType, + EdgeEventActionType cloudEventAction, EntityId entityId, + JsonNode entityBody) throws ExecutionException, InterruptedException { + saveCloudEventAsync(tenantId, cloudEventType, cloudEventAction, entityId, entityBody).get(); + } + + @Override + public ListenableFuture saveCloudEventAsync(TenantId tenantId, CloudEventType cloudEventType, + EdgeEventActionType cloudEventAction, EntityId entityId, + JsonNode entityBody) { + CloudEvent cloudEvent = new CloudEvent( + tenantId, + cloudEventAction, + entityId != null ? entityId.getId() : null, + cloudEventType, + entityBody + ); + return saveAsync(cloudEvent); + } + + @Override + public ListenableFuture saveAsync(CloudEvent cloudEvent) { + return saveCloudEventToTopic(cloudEvent, tbCloudEventProvider.getCloudEventMsgProducer()); + } + + @Override + public ListenableFuture saveTsKvAsync(CloudEvent cloudEvent) { + return saveCloudEventToTopic(cloudEvent, tbCloudEventProvider.getCloudEventTSMsgProducer()); + } + + private ListenableFuture saveCloudEventToTopic(CloudEvent cloudEvent, TbQueueProducer> producer) { + cloudEventValidator.validate(cloudEvent, CloudEvent::getTenantId); + log.trace("Save cloud event {}", cloudEvent); + SettableFuture futureToSet = SettableFuture.create(); + saveCloudEventToTopic(cloudEvent, producer, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + futureToSet.set(null); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to send cloud event", t); + futureToSet.setException(t); + } + }); + return futureToSet; + } + + private void saveCloudEventToTopic(CloudEvent cloudEvent, TbQueueProducer> producer, TbQueueCallback callback) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + + ToCloudEventMsg toCloudEventMsg = ToCloudEventMsg.newBuilder() + .setCloudEventMsg(ProtoUtils.toProto(cloudEvent)) + .build(); + + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCloudEventMsg), callback); + } + + @Override + public PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + throw new RuntimeException("Not implemented!"); + } + + @Override + public PageData findTsKvCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + throw new RuntimeException("Not implemented!"); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudManagerService.java b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudManagerService.java new file mode 100644 index 0000000000..cb8d881edc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cloud/KafkaCloudManagerService.java @@ -0,0 +1,153 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cloud; + +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.cloud.CloudEvent; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.provider.TbCloudEventQueueFactory; +import org.thingsboard.server.queue.settings.TbQueueCloudEventSettings; +import org.thingsboard.server.queue.settings.TbQueueCloudEventTSSettings; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +@Service +@Primary +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCloudManagerService extends BaseCloudManagerService { + + private QueueConsumerManager> consumer; + private QueueConsumerManager> tsConsumer; + + private ExecutorService consumerExecutor; + private ExecutorService tsConsumerExecutor; + + @Autowired + private TbCloudEventQueueFactory tbCloudEventQueueProvider; + + @Autowired + private TbQueueCloudEventTSSettings tbQueueCloudEventTSSettings; + + @Autowired + private TbQueueCloudEventSettings tbQueueCloudEventSettings; + + @Override + protected void launchUplinkProcessing() { + if (consumer == null) { + this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("cloud-event-consumer")); + this.consumer = QueueConsumerManager.>builder() + .name("TB Cloud Events") + .msgPackProcessor(this::processUplinkMessages) + .pollInterval(tbQueueCloudEventSettings.getPollInterval()) + .consumerCreator(tbCloudEventQueueProvider::createCloudEventMsgConsumer) + .consumerExecutor(consumerExecutor) + .threadPrefix("cloud-events") + .build(); + consumer.subscribe(); + consumer.launch(); + } + if (tsConsumer == null) { + this.tsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-cloud-event-consumer")); + this.tsConsumer = QueueConsumerManager.>builder() + .name("TB TS Cloud Events") + .msgPackProcessor(this::processTsUplinkMessages) + .pollInterval(tbQueueCloudEventTSSettings.getPollInterval()) + .consumerCreator(tbCloudEventQueueProvider::createCloudEventTSMsgConsumer) + .consumerExecutor(tsConsumerExecutor) + .threadPrefix("ts-cloud-events") + .build(); + tsConsumer.subscribe(); + tsConsumer.launch(); + } + } + + @Override + protected void resetQueueOffset() { + } + + @PreDestroy + private void onDestroy() throws InterruptedException { + super.destroy(); + + consumer.stop(); + consumerExecutor.shutdown(); + + tsConsumer.stop(); + tsConsumerExecutor.shutdown(); + } + + + private void processUplinkMessages(List> msgs, TbQueueConsumer> consumer) { + log.trace("[{}] starting processing edge events", tenantId); + if (initialized) { + isGeneralProcessInProgress = true; + processMessages(msgs, consumer); + isGeneralProcessInProgress = false; + } else { + sleep(); + } + + } + + private void processTsUplinkMessages(List> msgs, TbQueueConsumer> consumer) { + if (initialized && !isGeneralProcessInProgress) { + processMessages(msgs, consumer); + } else { + sleep(); + } + } + + private void sleep() { + try { + Thread.sleep(cloudEventStorageSettings.getNoRecordsSleepInterval()); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + + private void processMessages(List> msgs, TbQueueConsumer> consumer) { + List cloudEvents = new ArrayList<>(); + for (TbProtoQueueMsg msg : msgs) { + CloudEvent cloudEvent = ProtoUtils.fromProto(msg.getValue().getCloudEventMsg()); + cloudEvents.add(cloudEvent); + } + try { + boolean isInterrupted = processCloudEvents(cloudEvents, false).get(); + if (isInterrupted) { + log.debug("[{}] Send uplink messages task was interrupted", tenantId); + } else { + consumer.commit(); + } + } catch (Exception e) { + log.error("Failed to process all uplink messages", e); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cloud/BaseCloudEventService.java b/application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudEventService.java similarity index 52% rename from dao/src/main/java/org/thingsboard/server/dao/cloud/BaseCloudEventService.java rename to application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudEventService.java index c17b6dc8cc..9a3f76f45d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cloud/BaseCloudEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudEventService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.cloud; +package org.thingsboard.server.service.cloud; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; @@ -21,104 +21,86 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.cloud.CloudEvent; import org.thingsboard.server.common.data.cloud.CloudEventType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.edge.EdgeSettings; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cloud.CloudEventDao; +import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.TsKvCloudEventDao; import org.thingsboard.server.dao.service.DataValidator; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; -@Service +import static org.thingsboard.server.service.cloud.PostgresCloudManagerService.QUEUE_START_TS_ATTR_KEY; + @Slf4j +@Service @AllArgsConstructor -public class BaseCloudEventService implements CloudEventService { - - public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; +public class PostgresCloudEventService implements CloudEventService { private static final List CLOUD_EVENT_ACTION_WITHOUT_DUPLICATES = List.of( EdgeEventActionType.ATTRIBUTES_REQUEST, - EdgeEventActionType.RELATION_REQUEST); - - public CloudEventDao cloudEventDao; - public TsKvCloudEventDao tsKvCloudEventDao; - - public AttributesService attributesService; - - private DataValidator cloudEventValidator; - - @Override - public void cleanupEvents(long ttl) { - cloudEventDao.cleanupEvents(ttl); - tsKvCloudEventDao.cleanupEvents(ttl); - } - - @Override - public ListenableFuture saveAsync(CloudEvent cloudEvent) { - cloudEventValidator.validate(cloudEvent, CloudEvent::getTenantId); - return cloudEventDao.saveAsync(cloudEvent); - } + EdgeEventActionType.RELATION_REQUEST + ); - @Override - public ListenableFuture saveTsKvAsync(CloudEvent cloudEvent) { - cloudEventValidator.validate(cloudEvent, CloudEvent::getTenantId); - return tsKvCloudEventDao.saveAsync(cloudEvent); - } + private final AttributesService attributesService; + private final CloudEventDao cloudEventDao; + private final TsKvCloudEventDao tsKvCloudEventDao; + private final DataValidator cloudEventValidator; @Override - public void saveCloudEvent(TenantId tenantId, - CloudEventType cloudEventType, - EdgeEventActionType cloudEventAction, - EntityId entityId, - JsonNode entityBody, - Long queueStartTs) throws ExecutionException, InterruptedException { - saveCloudEventAsync(tenantId, cloudEventType, cloudEventAction, entityId, entityBody, queueStartTs).get(); + public void saveCloudEvent(TenantId tenantId, CloudEventType cloudEventType, + EdgeEventActionType cloudEventAction, EntityId entityId, + JsonNode entityBody) throws ExecutionException, InterruptedException { + saveCloudEventAsync(tenantId, cloudEventType, cloudEventAction, entityId, entityBody).get(); } @Override public ListenableFuture saveCloudEventAsync(TenantId tenantId, CloudEventType cloudEventType, EdgeEventActionType cloudEventAction, EntityId entityId, - JsonNode entityBody, Long queueStartTs) { - if (shouldAddEventToQueue(tenantId, cloudEventType, cloudEventAction, entityId, queueStartTs)) { - CloudEvent cloudEvent = new CloudEvent(); - cloudEvent.setTenantId(tenantId); - cloudEvent.setType(cloudEventType); - cloudEvent.setAction(cloudEventAction); - cloudEvent.setEntityId(entityId != null ? entityId.getId() : null); - cloudEvent.setEntityBody(entityBody); + JsonNode entityBody) { + if (shouldAddEventToQueue(tenantId, cloudEventType, cloudEventAction, entityId)) { + CloudEvent cloudEvent = new CloudEvent( + tenantId, + cloudEventAction, + entityId != null ? entityId.getId() : null, + cloudEventType, + entityBody + ); return saveAsync(cloudEvent); } else { return Futures.immediateFuture(null); } } - private boolean shouldAddEventToQueue(TenantId tenantId, CloudEventType cloudEventType, EdgeEventActionType cloudEventAction, - EntityId entityId, Long queueStartTs) { + private boolean shouldAddEventToQueue(TenantId tenantId, CloudEventType cloudEventType, + EdgeEventActionType cloudEventAction, EntityId entityId) { + Long queueStartTs = null; + try { + Optional attributeKvEntry = attributesService.find(tenantId, tenantId, AttributeScope.SERVER_SCOPE, QUEUE_START_TS_ATTR_KEY).get(); + if (attributeKvEntry.isPresent() && attributeKvEntry.get().getLongValue().isPresent()) { + queueStartTs = attributeKvEntry.get().getLongValue().get(); + } + } catch (Exception ignored) {} + if (queueStartTs == null || queueStartTs <= 0 || !CLOUD_EVENT_ACTION_WITHOUT_DUPLICATES.contains(cloudEventAction)) { return true; } long countMsgsInQueue = cloudEventDao.countEventsByTenantIdAndEntityIdAndActionAndTypeAndStartTimeAndEndTime( - tenantId.getId(), entityId.getId(), cloudEventType, cloudEventAction, queueStartTs, System.currentTimeMillis() - ); + tenantId.getId(), entityId.getId(), cloudEventType, cloudEventAction, queueStartTs, System.currentTimeMillis()); if (countMsgsInQueue > 0) { - log.info("{} Skipping adding of {} event because it's already present in db {} {}", - tenantId, cloudEventAction, entityId, cloudEventType); + log.info("{} Skipping adding of {} event because it's already present in db {} {}", tenantId, cloudEventAction, entityId, cloudEventType); return false; } @@ -126,45 +108,27 @@ private boolean shouldAddEventToQueue(TenantId tenantId, CloudEventType cloudEve } @Override - public PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return cloudEventDao.findCloudEvents(tenantId.getId(), seqIdStart, seqIdEnd, pageLink); + public ListenableFuture saveAsync(CloudEvent cloudEvent) { + cloudEventValidator.validate(cloudEvent, CloudEvent::getTenantId); + log.trace("Save cloud event {}", cloudEvent); + return cloudEventDao.saveAsync(cloudEvent); } @Override - public PageData findTsKvCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return tsKvCloudEventDao.findCloudEvents(tenantId.getId(), seqIdStart, seqIdEnd, pageLink); + public ListenableFuture saveTsKvAsync(CloudEvent cloudEvent) { + cloudEventValidator.validate(cloudEvent, CloudEvent::getTenantId); + + return tsKvCloudEventDao.saveAsync(cloudEvent); } @Override - public EdgeSettings findEdgeSettings(TenantId tenantId) { - try { - Optional attr = - attributesService.find(tenantId, tenantId, AttributeScope.SERVER_SCOPE, DataConstants.EDGE_SETTINGS_ATTR_KEY).get(); - if (attr.isPresent()) { - log.trace("Found current edge settings {}", attr.get().getValueAsString()); - return JacksonUtil.fromString(attr.get().getValueAsString(), EdgeSettings.class); - } else { - log.trace("Edge settings not found"); - return null; - } - } catch (Exception e) { - log.error("Exception while fetching edge settings", e); - throw new RuntimeException("Exception while fetching edge settings", e); - } + public PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + return cloudEventDao.findCloudEvents(tenantId.getId(), seqIdStart, seqIdEnd, pageLink); } @Override - public ListenableFuture> saveEdgeSettings(TenantId tenantId, EdgeSettings edgeSettings) { - try { - BaseAttributeKvEntry edgeSettingAttr = - new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_SETTINGS_ATTR_KEY, JacksonUtil.toString(edgeSettings)), System.currentTimeMillis()); - List attributes = - Collections.singletonList(edgeSettingAttr); - return attributesService.save(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attributes); - } catch (Exception e) { - log.error("Exception while saving edge settings", e); - throw new RuntimeException("Exception while saving edge settings", e); - } + public PageData findTsKvCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + return tsKvCloudEventDao.findCloudEvents(tenantId.getId(), seqIdStart, seqIdEnd, pageLink); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudManagerService.java b/application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudManagerService.java new file mode 100644 index 0000000000..92cea84394 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cloud/PostgresCloudManagerService.java @@ -0,0 +1,108 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cloud; + +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.service.cloud.rpc.CloudEventStorageSettings; + +import javax.annotation.PreDestroy; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + + +@Slf4j +@Service +@ConditionalOnExpression("'${queue.type:null}'!='kafka'") +public class PostgresCloudManagerService extends BaseCloudManagerService { + + @Autowired + private CloudEventStorageSettings cloudEventStorageSettings; + + @Autowired + private CloudEventService cloudEventService; + + private ExecutorService executor; + private ExecutorService tsExecutor; + + @PostConstruct + private void onInit() { + executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("postgres-cloud-manager")); + tsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("postgres-ts-cloud-manager")); + } + + @PreDestroy + private void onDestroy() throws InterruptedException { + super.destroy(); + if (executor != null) { + executor.shutdownNow(); + } + if (tsExecutor != null) { + tsExecutor.shutdownNow(); + } + } + + @Override + protected void launchUplinkProcessing() { + executor.submit(() -> launchUplinkProcessing(QUEUE_START_TS_ATTR_KEY, QUEUE_SEQ_ID_OFFSET_ATTR_KEY, true, + (TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) + -> cloudEventService.findCloudEvents(tenantId, seqIdStart, seqIdEnd, pageLink))); + tsExecutor.submit(() -> launchUplinkProcessing(QUEUE_TS_KV_START_TS_ATTR_KEY, QUEUE_TS_KV_SEQ_ID_OFFSET_ATTR_KEY, false, + (TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) + -> cloudEventService.findTsKvCloudEvents(tenantId, seqIdStart, seqIdEnd, pageLink))); + } + + private void launchUplinkProcessing(String queueStartTsAttrKey, String queueSeqIdAttrKey, boolean isGeneralMsg, CloudEventFinder finder) { + while (!Thread.interrupted()) { + try { + if (initialized) { + if (isGeneralMsg || !isGeneralProcessInProgress) { + Long queueSeqIdStart = getLongAttrByKey(tenantId, queueSeqIdAttrKey).get(); + TimePageLink pageLink = newCloudEventsAvailable(tenantId, queueSeqIdStart, queueStartTsAttrKey, finder); + + if (pageLink != null) { + processUplinkMessages(pageLink, queueSeqIdStart, queueStartTsAttrKey, queueSeqIdAttrKey, isGeneralMsg, finder); + } else { + log.trace("no new cloud events found for queue, isGeneralMsg = {}", isGeneralMsg); + sleep(); + } + } + } else { + TimeUnit.SECONDS.sleep(1); + } + } catch (Exception e) { + log.warn("Failed to process messages handling!", e); + } + } + } + + private void sleep() { + try { + Thread.sleep(cloudEventStorageSettings.getNoRecordsSleepInterval()); + } catch (InterruptedException e) { + log.error("Error during sleep", e); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/AssetCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/AssetCloudProcessor.java index 1650c7d47e..49540601e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/AssetCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/AssetCloudProcessor.java @@ -48,8 +48,7 @@ public class AssetCloudProcessor extends BaseAssetProcessor { public ListenableFuture processAssetMsgFromCloud(TenantId tenantId, - AssetUpdateMsg assetUpdateMsg, - Long queueStartTs) { + AssetUpdateMsg assetUpdateMsg) { AssetId assetId = new AssetId(new UUID(assetUpdateMsg.getIdMSB(), assetUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); @@ -57,8 +56,8 @@ public ListenableFuture processAssetMsgFromCloud(TenantId tenantId, switch (assetUpdateMsg.getMsgType()) { case ENTITY_CREATED_RPC_MESSAGE: case ENTITY_UPDATED_RPC_MESSAGE: - saveOrUpdateAsset(tenantId, assetId, assetUpdateMsg, queueStartTs); - return requestForAdditionalData(tenantId, assetId, queueStartTs); + saveOrUpdateAssetFromCloud(tenantId, assetId, assetUpdateMsg); + return requestForAdditionalData(tenantId, assetId); case ENTITY_DELETED_RPC_MESSAGE: Asset assetById = edgeCtx.getAssetService().findAssetById(tenantId, assetId); if (assetById != null) { @@ -75,7 +74,7 @@ public ListenableFuture processAssetMsgFromCloud(TenantId tenantId, } } - private void saveOrUpdateAsset(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg, Long queueStartTs) { + private void saveOrUpdateAssetFromCloud(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg) { Pair resultPair = super.saveOrUpdateAsset(tenantId, assetId, assetUpdateMsg); Boolean created = resultPair.getFirst(); if (created) { @@ -83,7 +82,7 @@ private void saveOrUpdateAsset(TenantId tenantId, AssetId assetId, AssetUpdateMs } Boolean assetNameUpdated = resultPair.getSecond(); if (assetNameUpdated) { - cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ASSET, EdgeEventActionType.UPDATED, assetId, null, queueStartTs); + cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ASSET, EdgeEventActionType.UPDATED, assetId, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/CustomerCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/CustomerCloudProcessor.java index 009387a568..313eaf2476 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/CustomerCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/CustomerCloudProcessor.java @@ -35,8 +35,7 @@ @Slf4j public class CustomerCloudProcessor extends BaseEdgeProcessor { - public ListenableFuture processCustomerMsgFromCloud(TenantId tenantId, CustomerUpdateMsg customerUpdateMsg, - Long queueStartTs) { + public ListenableFuture processCustomerMsgFromCloud(TenantId tenantId, CustomerUpdateMsg customerUpdateMsg) { CustomerId customerId = new CustomerId(new UUID(customerUpdateMsg.getIdMSB(), customerUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); @@ -54,7 +53,7 @@ public ListenableFuture processCustomerMsgFromCloud(TenantId tenantId, Cus } finally { customerCreationLock.unlock(); } - return requestForAdditionalData(tenantId, customerId, queueStartTs); + return requestForAdditionalData(tenantId, customerId); case ENTITY_DELETED_RPC_MESSAGE: Customer customerById = edgeCtx.getCustomerService().findCustomerById(tenantId, customerId); if (customerById != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DashboardCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DashboardCloudProcessor.java index 039d5b3dc9..591b1230c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DashboardCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DashboardCloudProcessor.java @@ -51,8 +51,7 @@ public class DashboardCloudProcessor extends BaseDashboardProcessor { public ListenableFuture processDashboardMsgFromCloud(TenantId tenantId, DashboardUpdateMsg dashboardUpdateMsg, - CustomerId customerId, - Long queueStartTs) { + CustomerId customerId) { DashboardId dashboardId = new DashboardId(new UUID(dashboardUpdateMsg.getIdMSB(), dashboardUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); @@ -63,7 +62,7 @@ public ListenableFuture processDashboardMsgFromCloud(TenantId tenantId, if (created) { pushDashboardCreatedEventToRuleEngine(tenantId, dashboardId); } - return requestForAdditionalData(tenantId, dashboardId, queueStartTs); + return requestForAdditionalData(tenantId, dashboardId); case ENTITY_DELETED_RPC_MESSAGE: Dashboard dashboardById = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); if (dashboardById != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DeviceCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DeviceCloudProcessor.java index 1f736b709d..8201d769cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DeviceCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/DeviceCloudProcessor.java @@ -64,16 +64,15 @@ public class DeviceCloudProcessor extends BaseDeviceProcessor { private TbCoreDeviceRpcService tbCoreDeviceRpcService; public ListenableFuture processDeviceMsgFromCloud(TenantId tenantId, - DeviceUpdateMsg deviceUpdateMsg, - Long queueStartTs) { + DeviceUpdateMsg deviceUpdateMsg) { DeviceId deviceId = new DeviceId(new UUID(deviceUpdateMsg.getIdMSB(), deviceUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); switch (deviceUpdateMsg.getMsgType()) { case ENTITY_CREATED_RPC_MESSAGE: case ENTITY_UPDATED_RPC_MESSAGE: - saveOrUpdateDevice(tenantId, deviceId, deviceUpdateMsg, queueStartTs); - return requestForAdditionalData(tenantId, deviceId, queueStartTs); + saveOrUpdateDeviceFromCloud(tenantId, deviceId, deviceUpdateMsg); + return requestForAdditionalData(tenantId, deviceId); case ENTITY_DELETED_RPC_MESSAGE: Device deviceById = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceId); if (deviceById != null) { @@ -90,7 +89,7 @@ public ListenableFuture processDeviceMsgFromCloud(TenantId tenantId, } } - private void saveOrUpdateDevice(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg, Long queueStartTs) { + private void saveOrUpdateDeviceFromCloud(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg) { Pair resultPair = super.saveOrUpdateDevice(tenantId, deviceId, deviceUpdateMsg); Boolean created = resultPair.getFirst(); if (created) { @@ -98,7 +97,7 @@ private void saveOrUpdateDevice(TenantId tenantId, DeviceId deviceId, DeviceUpda } Boolean deviceNameUpdated = resultPair.getSecond(); if (deviceNameUpdated) { - cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.DEVICE, EdgeEventActionType.UPDATED, deviceId, null, queueStartTs); + cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.DEVICE, EdgeEventActionType.UPDATED, deviceId, null); } } @@ -206,7 +205,7 @@ private void reply(ToDeviceRpcRequest rpcRequest, int requestId, FromDeviceRpcRe body.put("response", response.getResponse().orElse("{}")); } cloudEventService.saveCloudEvent(rpcRequest.getTenantId(), CloudEventType.DEVICE, EdgeEventActionType.RPC_CALL, - rpcRequest.getDeviceId(), body, 0L); + rpcRequest.getDeviceId(), body); } catch (Exception e) { log.debug("Can't process RPC response [{}] [{}]", rpcRequest, response, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/EntityViewCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/EntityViewCloudProcessor.java index 3552bc55c7..67c1658731 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/EntityViewCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/EntityViewCloudProcessor.java @@ -54,8 +54,7 @@ public class EntityViewCloudProcessor extends BaseEntityViewProcessor { private EntityViewMsgConstructorFactory entityViewMsgConstructorFactory; public ListenableFuture processEntityViewMsgFromCloud(TenantId tenantId, - EntityViewUpdateMsg entityViewUpdateMsg, - Long queueStartTs) { + EntityViewUpdateMsg entityViewUpdateMsg) { EntityViewId entityViewId = new EntityViewId(new UUID(entityViewUpdateMsg.getIdMSB(), entityViewUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); @@ -63,8 +62,8 @@ public ListenableFuture processEntityViewMsgFromCloud(TenantId tenantId, switch (entityViewUpdateMsg.getMsgType()) { case ENTITY_CREATED_RPC_MESSAGE: case ENTITY_UPDATED_RPC_MESSAGE: - saveOrUpdateEntityView(tenantId, entityViewId, entityViewUpdateMsg, queueStartTs); - return requestForAdditionalData(tenantId, entityViewId, queueStartTs); + saveOrUpdateEntityViewFromCloud(tenantId, entityViewId, entityViewUpdateMsg); + return requestForAdditionalData(tenantId, entityViewId); case ENTITY_DELETED_RPC_MESSAGE: EntityView entityViewById = edgeCtx.getEntityViewService().findEntityViewById(tenantId, entityViewId); if (entityViewById != null) { @@ -81,7 +80,7 @@ public ListenableFuture processEntityViewMsgFromCloud(TenantId tenantId, } } - private void saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg, Long queueStartTs) { + private void saveOrUpdateEntityViewFromCloud(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg) { Pair resultPair = super.saveOrUpdateEntityView(tenantId, entityViewId, entityViewUpdateMsg); Boolean created = resultPair.getFirst(); if (created) { @@ -89,7 +88,7 @@ private void saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId } Boolean entityViewNameUpdated = resultPair.getSecond(); if (entityViewNameUpdated) { - cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ENTITY_VIEW, EdgeEventActionType.UPDATED, entityViewId, null, queueStartTs); + cloudEventService.saveCloudEventAsync(tenantId, CloudEventType.ENTITY_VIEW, EdgeEventActionType.UPDATED, entityViewId, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/TenantCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/TenantCloudProcessor.java index 417bf7078b..625070d9b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/TenantCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/TenantCloudProcessor.java @@ -43,7 +43,7 @@ public class TenantCloudProcessor extends BaseEdgeProcessor { @Autowired private ApiUsageStateService apiUsageStateService; - public void createTenantIfNotExists(TenantId tenantId, Long queueStartTs) throws Exception { + public void createTenantIfNotExists(TenantId tenantId) throws Exception { try { cloudSynchronizationManager.getSync().set(true); Tenant tenant = edgeCtx.getTenantService().findTenantById(tenantId); @@ -60,7 +60,7 @@ public void createTenantIfNotExists(TenantId tenantId, Long queueStartTs) throws apiUsageStateService.createDefaultApiUsageState(savedTenant.getId(), null); } - requestForAdditionalData(tenantId, tenantId, queueStartTs).get(); + requestForAdditionalData(tenantId, tenantId).get(); } finally { cloudSynchronizationManager.getSync().remove(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/UserCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/UserCloudProcessor.java index cd312971f7..8506fcf8ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/UserCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/UserCloudProcessor.java @@ -41,8 +41,7 @@ public class UserCloudProcessor extends BaseEdgeProcessor { private final Lock userCreationLock = new ReentrantLock(); public ListenableFuture processUserMsgFromCloud(TenantId tenantId, - UserUpdateMsg userUpdateMsg, - Long queueStartTs) { + UserUpdateMsg userUpdateMsg) { UserId userId = new UserId(new UUID(userUpdateMsg.getIdMSB(), userUpdateMsg.getIdLSB())); try { cloudSynchronizationManager.getSync().set(true); @@ -64,7 +63,7 @@ public ListenableFuture processUserMsgFromCloud(TenantId tenantId, } finally { userCreationLock.unlock(); } - return requestForAdditionalData(tenantId, userId, queueStartTs); + return requestForAdditionalData(tenantId, userId); case ENTITY_DELETED_RPC_MESSAGE: User userToDelete = edgeCtx.getUserService().findUserById(tenantId, userId); if (userToDelete != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 7c37b0ac54..8f81408cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -103,6 +103,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiConsumer; + @Slf4j @Data public abstract class EdgeGrpcSession implements Closeable { @@ -111,7 +112,7 @@ public abstract class EdgeGrpcSession implements Closeable { private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; private static final int MAX_DOWNLINK_ATTEMPTS = 3; - private static final String RATE_LIMIT_REACHED = "Rate limit reached"; + public static final String RATE_LIMIT_REACHED = "Rate limit reached"; protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index f46850735b..1f0e00c402 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -334,18 +334,18 @@ protected boolean isEntityExists(TenantId tenantId, EntityId entityId) { return entityDaoRegistry.getDao(entityId.getEntityType()).existsById(tenantId, entityId.getId()); } - protected ListenableFuture requestForAdditionalData(TenantId tenantId, EntityId entityId, Long queueStartTs) { + protected ListenableFuture requestForAdditionalData(TenantId tenantId, EntityId entityId) { List> futures = new ArrayList<>(); CloudEventType cloudEventType = CloudUtils.getCloudEventTypeByEntityType(entityId.getEntityType()); log.info("Adding ATTRIBUTES_REQUEST/RELATION_REQUEST {} {}", entityId, cloudEventType); futures.add(cloudEventService.saveCloudEventAsync(tenantId, cloudEventType, - EdgeEventActionType.ATTRIBUTES_REQUEST, entityId, null, queueStartTs)); + EdgeEventActionType.ATTRIBUTES_REQUEST, entityId, null)); futures.add(cloudEventService.saveCloudEventAsync(tenantId, cloudEventType, - EdgeEventActionType.RELATION_REQUEST, entityId, null, queueStartTs)); + EdgeEventActionType.RELATION_REQUEST, entityId, null)); if (CloudEventType.DEVICE.equals(cloudEventType) || CloudEventType.ASSET.equals(cloudEventType)) { futures.add(cloudEventService.saveCloudEventAsync(tenantId, cloudEventType, - EdgeEventActionType.ENTITY_VIEW_REQUEST, entityId, null, queueStartTs)); + EdgeEventActionType.ENTITY_VIEW_REQUEST, entityId, null)); } return Futures.transform(Futures.allAsList(futures), voids -> null, dbCallbackExecutorService); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index fa3888bbf1..8cee882844 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.EdgeSettingsService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.tenant.TenantService; @@ -75,7 +75,7 @@ public class DefaultDataUpdateService implements DataUpdateService { private TenantService tenantService; @Autowired - private CloudEventService cloudEventService; + private EdgeSettingsService edgeSettingsService; @Autowired private WidgetsBundleService widgetsBundleService; @@ -270,10 +270,10 @@ protected PageData findEntities(String region, PageLink pageLink) { @Override protected void updateEntity(Tenant tenant) { try { - EdgeSettings edgeSettings = cloudEventService.findEdgeSettings(tenant.getId()); + EdgeSettings edgeSettings = edgeSettingsService.findEdgeSettings(tenant.getId()); if (edgeSettings != null) { edgeSettings.setFullSyncRequired(true); - cloudEventService.saveEdgeSettings(tenant.getId(), edgeSettings); + edgeSettingsService.saveEdgeSettings(tenant.getId(), edgeSettings); } else { log.warn("Edge settings not found for tenant: {}", tenant.getId()); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/cloud/CloudEventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/cloud/CloudEventsCleanUpService.java index 40ac25f6c7..91ba854f39 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/cloud/CloudEventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/cloud/CloudEventsCleanUpService.java @@ -20,7 +20,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.cloud.CloudEventService; +import org.thingsboard.server.dao.cloud.CloudEventDao; +import org.thingsboard.server.dao.cloud.TsKvCloudEventDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.queue.discovery.PartitionService; @@ -32,7 +33,7 @@ @TbCoreComponent @Slf4j @Service -@ConditionalOnExpression("${sql.ttl.cloud_events.enabled:true} && ${sql.ttl.cloud_events.cloud_events_ttl:0} > 0") +@ConditionalOnExpression("${sql.ttl.cloud_events.enabled:true} && ${sql.ttl.cloud_events.cloud_events_ttl:0} > 0 && '${queue.type:null}' != 'kafka'") public class CloudEventsCleanUpService extends AbstractCleanUpService { public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION = @@ -47,13 +48,19 @@ public class CloudEventsCleanUpService extends AbstractCleanUpService { @Value("${sql.ttl.cloud_events.enabled}") private boolean ttlTaskExecutionEnabled; - private final CloudEventService cloudEventService; + private final CloudEventDao cloudEventDao; + private final TsKvCloudEventDao tsKvCloudEventDao; private final SqlPartitioningRepository partitioningRepository; - public CloudEventsCleanUpService(PartitionService partitionService, CloudEventService cloudEventService, SqlPartitioningRepository partitioningRepository) { + + public CloudEventsCleanUpService(PartitionService partitionService, + CloudEventDao cloudEventDao, + TsKvCloudEventDao tsKvCloudEventDao, + SqlPartitioningRepository partitioningRepository) { super(partitionService); - this.cloudEventService = cloudEventService; + this.cloudEventDao = cloudEventDao; + this.tsKvCloudEventDao = tsKvCloudEventDao; this.partitioningRepository = partitioningRepository; } @@ -61,7 +68,8 @@ public CloudEventsCleanUpService(PartitionService partitionService, CloudEventSe public void cleanUp() { long cloudEventsExpTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttl); if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { - cloudEventService.cleanupEvents(cloudEventsExpTime); + cloudEventDao.cleanupEvents(cloudEventsExpTime); + tsKvCloudEventDao.cleanupEvents(cloudEventsExpTime); } else { // clean up 'cloud_event' partitioningRepository.cleanupPartitionsCache(ModelConstants.CLOUD_EVENT_COLUMN_FAMILY_NAME, cloudEventsExpTime, TimeUnit.HOURS.toMillis(partitionSizeInHours)); diff --git a/application/src/main/resources/tb-edge.yml b/application/src/main/resources/tb-edge.yml index ebfc86960b..5d69a2c406 100644 --- a/application/src/main/resources/tb-edge.yml +++ b/application/src/main/resources/tb-edge.yml @@ -1649,6 +1649,16 @@ queue: - key: max.poll.records # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + tb_cloud_event: + # Consumer properties for Kafka tb_cloud_event topic + - key: max.poll.records + # Amount of records to be returned in a single poll for cloud event topic + value: "${TB_QUEUE_KAFKA_CLOUD_EVENT_MAX_POLL_RECORDS:50}" + tb_cloud_event_ts: + # Consumer properties for Kafka tb_cloud_event_ts topic + - key: max.poll.records + # Amount of records to be returned in a single poll for cloud event timeseries topic + value: "${TB_QUEUE_KAFKA_CLOUD_EVENT_TS_MAX_POLL_RECORDS:50}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1678,6 +1688,10 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for CloudEvent topic + cloud_event: "${TB_QUEUE_KAFKA_CLOUD_EVENT_TOPIC_PROPERTIES:retention.ms:2678400000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for TsCloudEvent topic + cloud_event_ts: "${TB_QUEUE_KAFKA_CLOUD_EVENT_TS_TOPIC_PROPERTIES:retention.ms:2678400000;segment.bytes:52428800;retention.bytes:10485760000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1937,6 +1951,16 @@ queue: enabled: "${TB_QUEUE_EDGE_STATS_ENABLED:true}" # Statistics printing interval for Edge services print-interval-ms: "${TB_QUEUE_EDGE_STATS_PRINT_INTERVAL_MS:60000}" + cloud-event: + # Default topic name for Kafka, RabbitMQ, etc. + topic: "${TB_QUEUE_CLOUD_EVENT_TOPIC:tb_cloud_event}" + # Poll interval for topics related to Cloud Event services + poll-interval: "${TB_QUEUE_CLOUD_EVENT_POLL_INTERVAL_MS:25}" + cloud-event-ts: + # Default topic name for Kafka, RabbitMQ, etc. + topic: "${TB_QUEUE_CLOUD_EVENT_TS_TOPIC:tb_cloud_event_ts}" + # Poll interval for topics related to Cloud Event services + poll-interval: "${TB_QUEUE_CLOUD_EVENT_TS_POLL_INTERVAL_MS:25}" # Event configuration parameters event: diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/CloudEventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/CloudEventService.java index cdd259bbc7..9fe4138d60 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/CloudEventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/CloudEventService.java @@ -20,43 +20,33 @@ import org.thingsboard.server.common.data.cloud.CloudEvent; import org.thingsboard.server.common.data.cloud.CloudEventType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.edge.EdgeSettings; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import java.util.List; import java.util.concurrent.ExecutionException; public interface CloudEventService { - ListenableFuture saveAsync(CloudEvent cloudEvent); - - ListenableFuture saveTsKvAsync(CloudEvent cloudEvent); - void saveCloudEvent(TenantId tenantId, CloudEventType cloudEventType, EdgeEventActionType cloudEventAction, EntityId entityId, - JsonNode entityBody, - Long queueStartTs) throws ExecutionException, InterruptedException; + JsonNode entityBody) throws ExecutionException, InterruptedException; ListenableFuture saveCloudEventAsync(TenantId tenantId, CloudEventType cloudEventType, EdgeEventActionType cloudEventAction, EntityId entityId, - JsonNode entityBody, - Long queueStartTs); - - PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); + JsonNode entityBody); - PageData findTsKvCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); + ListenableFuture saveAsync(CloudEvent cloudEvent); - EdgeSettings findEdgeSettings(TenantId tenantId); + ListenableFuture saveTsKvAsync(CloudEvent cloudEvent); - ListenableFuture> saveEdgeSettings(TenantId tenantId, EdgeSettings edgeSettings); + PageData findCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); - void cleanupEvents(long ttl); + PageData findTsKvCloudEvents(TenantId tenantId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/GeneralUplinkMessageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/EdgeSettingsService.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/cloud/GeneralUplinkMessageService.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/EdgeSettingsService.java index c2a99adfa0..0492523439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/GeneralUplinkMessageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cloud/EdgeSettingsService.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cloud; +package org.thingsboard.server.dao.cloud; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edge.EdgeSettings; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.TimePageLink; -public interface GeneralUplinkMessageService extends UplinkMessageService { +import java.util.List; - TimePageLink newCloudEventsAvailable(TenantId tenantId, Long queueSeqIdStart); +public interface EdgeSettingsService { - ListenableFuture getQueueStartTs(TenantId tenantId); + EdgeSettings findEdgeSettings(TenantId tenantId); + ListenableFuture> saveEdgeSettings(TenantId tenantId, EdgeSettings edgeSettings); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cloud/CloudEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/cloud/CloudEvent.java index 73730e7687..30028a6e64 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cloud/CloudEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cloud/CloudEvent.java @@ -45,4 +45,12 @@ public CloudEvent() { public CloudEvent(CloudEventId id) { super(id); } -} \ No newline at end of file + + public CloudEvent(TenantId tenantId, EdgeEventActionType action, UUID entityId, CloudEventType type, JsonNode entityBody) { + this.tenantId = tenantId; + this.action = action; + this.entityId = entityId; + this.type = type; + this.entityBody = entityBody; + } +} diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 264b23f118..5854663f82 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cloud.CloudEvent; +import org.thingsboard.server.common.data.cloud.CloudEventType; import org.thingsboard.server.common.data.device.data.CoapDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.PowerMode; @@ -225,6 +227,43 @@ public static EdgeEvent fromProto(TransportProtos.EdgeEventMsgProto proto) { return edgeEvent; } + public static TransportProtos.CloudEventMsgProto toProto(CloudEvent cloudEvent) { + TransportProtos.CloudEventMsgProto.Builder builder = TransportProtos.CloudEventMsgProto.newBuilder(); + + builder.setTenantIdMSB(cloudEvent.getTenantId().getId().getMostSignificantBits()); + builder.setTenantIdLSB(cloudEvent.getTenantId().getId().getLeastSignificantBits()); + builder.setEntityType(cloudEvent.getType().name()); + builder.setAction(cloudEvent.getAction().name()); + + if (cloudEvent.getEntityId() != null) { + builder.setEntityIdMSB(cloudEvent.getEntityId().getMostSignificantBits()); + builder.setEntityIdLSB(cloudEvent.getEntityId().getLeastSignificantBits()); + } + if (cloudEvent.getEntityBody() != null) { + builder.setEntityBody(JacksonUtil.toString(cloudEvent.getEntityBody())); + } + + return builder.build(); + } + + public static CloudEvent fromProto(TransportProtos.CloudEventMsgProto proto) { + CloudEvent cloudEvent = new CloudEvent(); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + cloudEvent.setTenantId(tenantId); + cloudEvent.setType(CloudEventType.valueOf(proto.getEntityType())); + cloudEvent.setAction(EdgeEventActionType.valueOf(proto.getAction())); + + if (proto.hasEntityIdMSB() && proto.hasEntityIdLSB()) { + cloudEvent.setEntityId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } + + if (proto.hasEntityBody()) { + cloudEvent.setEntityBody(JacksonUtil.toJsonNode(proto.getEntityBody())); + } + + return cloudEvent; + } + public static TransportProtos.EdgeHighPriorityMsgProto toProto(EdgeHighPriorityMsg msg) { TransportProtos.EdgeHighPriorityMsgProto.Builder builder = TransportProtos.EdgeHighPriorityMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 584757b097..3c13c820ee 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1157,6 +1157,16 @@ message EdgeNotificationMsgProto { int64 originatorEdgeIdLSB = 14; } +message CloudEventMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + string action = 4; + optional int64 entityIdMSB = 5; + optional int64 entityIdLSB = 6; + optional string entityBody = 7; +} + message EdgeHighPriorityMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1554,6 +1564,10 @@ message ToEdgeMsg { EdgeNotificationMsgProto edgeNotificationMsg = 1; } +message ToCloudEventMsg { + CloudEventMsgProto cloudEventMsg = 1; +} + message ToEdgeNotificationMsg { EdgeHighPriorityMsgProto edgeHighPriority = 1; EdgeEventUpdateMsgProto edgeEventUpdate = 2; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..4efd132598 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,10 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.cloud_event:}") + private String cloudEventProperties; + @Value("${queue.kafka.topic-properties.cloud_event_ts:}") + private String cloudEventTSProperties; @Getter private Map coreConfigs; @@ -79,6 +83,10 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map cloudEventConfigs; + @Getter + private Map cloudEventTSConfigs; @PostConstruct private void init() { @@ -97,6 +105,8 @@ private void init() { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + cloudEventConfigs = PropertyUtils.getProps(cloudEventProperties); + cloudEventTSConfigs = PropertyUtils.getProps(cloudEventTSProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..12b972f8b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -54,6 +55,8 @@ import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCloudEventSettings; +import org.thingsboard.server.queue.settings.TbQueueCloudEventTSSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -67,7 +70,7 @@ @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory, TbCloudEventQueueFactory { private final TopicService topicService; private final TbKafkaSettings kafkaSettings; @@ -81,6 +84,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueEdgeSettings edgeSettings; private final TbKafkaConsumerStatsService consumerStatsService; + private final TbQueueCloudEventSettings cloudEventSettings; + private final TbQueueCloudEventTSSettings cloudEventTSSettings; + private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorRequestAdmin; @@ -95,6 +101,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cloudEventAdmin; + private final TbQueueAdmin cloudEventTSAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -107,7 +116,9 @@ public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafk TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + TbQueueCloudEventSettings cloudEventSettings, + TbQueueCloudEventTSSettings cloudEventTSSettings) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -119,6 +130,8 @@ public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafk this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.cloudEventSettings = cloudEventSettings; + this.cloudEventTSSettings = cloudEventTSSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +146,8 @@ public KafkaMonolithQueueFactory(TopicService topicService, TbKafkaSettings kafk this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cloudEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventConfigs()); + this.cloudEventTSAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventTSConfigs()); } @Override @@ -490,6 +505,60 @@ public TbQueueProducer> createEdgeEv return requestBuilder.build(); } + @Override + public TbQueueConsumer> createCloudEventMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventSettings.getTopic())); + consumerBuilder.clientId("monolith-cloud-event-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-cloud-event-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()))); + consumerBuilder.admin(cloudEventAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-to-cloud-event-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventSettings.getTopic())); + requestBuilder.admin(cloudEventAdmin); + + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCloudEventTSMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + consumerBuilder.clientId("monolith-cloud-event-ts-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-cloud-event-ts-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()))); + consumerBuilder.admin(cloudEventTSAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventTSMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-to-cloud-event-ts-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + requestBuilder.admin(cloudEventTSAdmin); + + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -522,5 +591,12 @@ private void destroy() { if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cloudEventAdmin != null) { + cloudEventAdmin.destroy(); + } + if (cloudEventTSAdmin != null) { + cloudEventTSAdmin.destroy(); + } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..e0a3a06aac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -53,6 +54,8 @@ import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCloudEventSettings; +import org.thingsboard.server.queue.settings.TbQueueCloudEventTSSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -66,7 +69,7 @@ @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") -public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { +public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory, TbCloudEventQueueFactory { private final TopicService topicService; private final TbKafkaSettings kafkaSettings; @@ -80,6 +83,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCloudEventSettings cloudEventSettings; + private final TbQueueCloudEventTSSettings cloudEventTSSettings; + private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorRequestAdmin; @@ -94,6 +100,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cloudEventAdmin; + private final TbQueueAdmin cloudEventTSAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbCoreQueueFactory(TopicService topicService, @@ -107,7 +116,9 @@ public KafkaTbCoreQueueFactory(TopicService topicService, TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + TbQueueCloudEventSettings cloudEventSettings, + TbQueueCloudEventTSSettings cloudEventTSSettings) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -119,6 +130,8 @@ public KafkaTbCoreQueueFactory(TopicService topicService, this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.cloudEventSettings = cloudEventSettings; + this.cloudEventTSSettings = cloudEventTSSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +146,8 @@ public KafkaTbCoreQueueFactory(TopicService topicService, this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cloudEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventConfigs()); + this.cloudEventTSAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventTSConfigs()); } @Override @@ -439,6 +454,60 @@ public TbQueueProducer> createEdgeEv return requestBuilder.build(); } + @Override + public TbQueueConsumer> createCloudEventMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventSettings.getTopic())); + consumerBuilder.clientId("tb-core-cloud-event-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-core-cloud-event-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cloudEventAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-cloud-event" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventSettings.getTopic())); + requestBuilder.admin(cloudEventAdmin); + + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCloudEventTSMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + consumerBuilder.clientId("tb-core-cloud-event-ts-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-core-cloud-event-ts-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cloudEventTSAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventTSMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-cloud-event-ts" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + requestBuilder.admin(cloudEventTSAdmin); + + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -468,5 +537,12 @@ private void destroy() { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cloudEventAdmin != null) { + cloudEventAdmin.destroy(); + } + if (cloudEventTSAdmin != null) { + cloudEventTSAdmin.destroy(); + } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..2a2c740f74 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -49,6 +50,8 @@ import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCloudEventSettings; +import org.thingsboard.server.queue.settings.TbQueueCloudEventTSSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -60,7 +63,7 @@ @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") -public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { +public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory, TbCloudEventQueueFactory { private final TopicService topicService; private final TbKafkaSettings kafkaSettings; @@ -72,6 +75,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCloudEventSettings cloudEventSettings; + private final TbQueueCloudEventTSSettings cloudEventTSSettings; + private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorRequestAdmin; @@ -81,6 +87,10 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + + private final TbQueueAdmin cloudEventAdmin; + private final TbQueueAdmin cloudEventTSAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -91,7 +101,9 @@ public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueEdgeSettings edgeSettings, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + TbQueueCloudEventSettings cloudEventSettings, + TbQueueCloudEventTSSettings cloudEventTSSettings) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -101,6 +113,8 @@ public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.cloudEventSettings = cloudEventSettings; + this.cloudEventTSSettings = cloudEventTSSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -111,6 +125,8 @@ public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cloudEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventConfigs()); + this.cloudEventTSAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCloudEventTSConfigs()); } @Override @@ -293,6 +309,60 @@ public TbQueueProducer> createHousekeep .build(); } + @Override + public TbQueueConsumer> createCloudEventMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventSettings.getTopic())); + consumerBuilder.clientId("tb-rule-engine-cloud-event-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-cloud-event-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cloudEventAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-cloud-event-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventSettings.getTopic())); + requestBuilder.admin(cloudEventAdmin); + + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCloudEventTSMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + consumerBuilder.clientId("tb-rule-engine-cloud-event-ts-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-cloud-event-ts-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCloudEventMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cloudEventTSAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCloudEventTSMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-cloud-event-ts-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(cloudEventTSSettings.getTopic())); + requestBuilder.admin(cloudEventTSAdmin); + + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -313,5 +383,12 @@ private void destroy() { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cloudEventAdmin != null) { + cloudEventAdmin.destroy(); + } + if (cloudEventTSAdmin != null) { + cloudEventTSAdmin.destroy(); + } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventProvider.java new file mode 100644 index 0000000000..78547dab9b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventProvider.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for providing various Producers to other services. + */ +public interface TbCloudEventProvider { + TbQueueProducer> getCloudEventMsgProducer(); + + TbQueueProducer> getCloudEventTSMsgProducer(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventQueueFactory.java new file mode 100644 index 0000000000..769942dab8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCloudEventQueueFactory.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by KafkaCloudEventService. + */ +public interface TbCloudEventQueueFactory { + TbQueueConsumer> createCloudEventMsgConsumer(); + + TbQueueProducer> createCloudEventMsgProducer(); + + TbQueueConsumer> createCloudEventTSMsgConsumer(); + + TbQueueProducer> createCloudEventTSMsgProducer(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreCloudEventProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreCloudEventProvider.java new file mode 100644 index 0000000000..d00406ee10 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreCloudEventProvider.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import jakarta.annotation.PostConstruct; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCloudEventMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class TbCoreCloudEventProvider implements TbCloudEventProvider { + + private final TbCloudEventQueueFactory tbCloudEventQueueProvider; + private TbQueueProducer> toCloudEventProducer; + private TbQueueProducer> toCloudEventTSProducer; + + public TbCoreCloudEventProvider(TbCloudEventQueueFactory tbCloudEventQueueProvider) { + this.tbCloudEventQueueProvider = tbCloudEventQueueProvider; + } + + @PostConstruct + public void init() { + toCloudEventProducer = tbCloudEventQueueProvider.createCloudEventMsgProducer(); + toCloudEventTSProducer = tbCloudEventQueueProvider.createCloudEventTSMsgProducer(); + } + + @Override + public TbQueueProducer> getCloudEventMsgProducer() { + return toCloudEventProducer; + } + + @Override + public TbQueueProducer> getCloudEventTSMsgProducer() { + return toCloudEventTSProducer; + } +} + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventSettings.java new file mode 100644 index 0000000000..fdf98c17da --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventSettings.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCloudEventSettings { + + @Value("${queue.cloud-event.topic}") + private String topic; + @Value("${queue.cloud-event.poll-interval}") + private Long pollInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventTSSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventTSSettings.java new file mode 100644 index 0000000000..1a6b392cc8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCloudEventTSSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCloudEventTSSettings { + + @Value("${queue.cloud-event-ts.topic}") + private String topic; + @Value("${queue.cloud-event-ts.poll-interval}") + private Long pollInterval; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cloud/DefaultEdgeSettingsService.java b/dao/src/main/java/org/thingsboard/server/dao/cloud/DefaultEdgeSettingsService.java new file mode 100644 index 0000000000..e414db457d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cloud/DefaultEdgeSettingsService.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cloud; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.edge.EdgeSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.attributes.AttributesService; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultEdgeSettingsService implements EdgeSettingsService { + + private static final String FETCHING_EDGE_SETTINGS_ERROR_MESSAGE = "Fetching edge settings failed"; + private final AttributesService attributesService; + + @Override + public EdgeSettings findEdgeSettings(TenantId tenantId) { + try { + Optional attr = + attributesService.find(tenantId, tenantId, AttributeScope.SERVER_SCOPE, DataConstants.EDGE_SETTINGS_ATTR_KEY).get(); + + if (attr.isPresent()) { + log.trace("Found current edge settings {}", attr.get().getValueAsString()); + return JacksonUtil.fromString(attr.get().getValueAsString(), EdgeSettings.class); + } else { + log.trace("Edge settings not found"); + return null; + } + } catch (Exception e) { + log.error(FETCHING_EDGE_SETTINGS_ERROR_MESSAGE, e); + throw new RuntimeException(FETCHING_EDGE_SETTINGS_ERROR_MESSAGE, e); + } + } + + @Override + public ListenableFuture> saveEdgeSettings(TenantId tenantId, EdgeSettings edgeSettings) { + StringDataEntry dataEntry = new StringDataEntry(DataConstants.EDGE_SETTINGS_ATTR_KEY, JacksonUtil.toString(edgeSettings)); + BaseAttributeKvEntry edgeSettingAttr = new BaseAttributeKvEntry(dataEntry, System.currentTimeMillis()); + List attributes = Collections.singletonList(edgeSettingAttr); + + return attributesService.save(tenantId, tenantId, AttributeScope.SERVER_SCOPE, attributes); + } + +} diff --git a/docker-edge/.env b/docker-edge/.env index 74e076e56f..44758df43d 100644 --- a/docker-edge/.env +++ b/docker-edge/.env @@ -1,7 +1,7 @@ DOCKER_REPO=thingsboard TB_NODE_DOCKER_NAME=tb-node -TB_VERSION=3.9.0-SNAPSHOT +TB_VERSION=3.9.0-RC TB_EDGE_DOCKER_NAME=tb-edge TB_EDGE_VERSION=3.9.0EDGE-RC diff --git a/docker-edge/docker-compose.postgres.yml b/docker-edge/docker-compose.postgres.yml index 7fc2493c99..8daa940367 100644 --- a/docker-edge/docker-compose.postgres.yml +++ b/docker-edge/docker-compose.postgres.yml @@ -23,7 +23,7 @@ services: ports: - "5432" environment: - POSTGRES_MULTIPLE_DATABASES: '"thingsboard","tb_edge"' + POSTGRES_MULTIPLE_DATABASES: '"thingsboard","tb_edge_1","tb_edge_2"' POSTGRES_PASSWORD: postgres volumes: - ./tb-node/postgres:/var/lib/postgresql/data @@ -34,7 +34,12 @@ services: - tb-node.env depends_on: - postgres - tb-edge: + tb-edge-1: + env_file: + - tb-edge.env + depends_on: + - postgres + tb-edge-2: env_file: - tb-edge.env depends_on: diff --git a/docker-edge/docker-compose.volumes.yml b/docker-edge/docker-compose.volumes.yml index 0007eb27e0..48f4ec9cb1 100644 --- a/docker-edge/docker-compose.volumes.yml +++ b/docker-edge/docker-compose.volumes.yml @@ -23,10 +23,14 @@ services: tb-monolith: volumes: - tb-log-volume:/var/log/thingsboard - tb-edge: + tb-edge-1: volumes: - - tb-edge-log-volume:/var/log/tb-edge - - tb-edge-data-volume:/data + - tb-edge-log-volume-1:/var/log/tb-edge + - tb-edge-data-volume-1:/data + tb-edge-2: + volumes: + - tb-edge-log-volume-2:/var/log/tb-edge + - tb-edge-data-volume-2:/data volumes: postgres-db-volume: external: @@ -34,9 +38,15 @@ volumes: tb-log-volume: external: name: ${TB_LOG_VOLUME} - tb-edge-log-volume: + tb-edge-log-volume-1: + external: + name: ${TB_EDGE_LOG_VOLUME_1} + tb-edge-data-volume-1: + external: + name: ${TB_EDGE_DATA_VOLUME_1} + tb-edge-log-volume-2: external: - name: ${TB_EDGE_LOG_VOLUME} - tb-edge-data-volume: + name: ${TB_EDGE_LOG_VOLUME_2} + tb-edge-data-volume-2: external: - name: ${TB_EDGE_DATA_VOLUME} + name: ${TB_EDGE_DATA_VOLUME_2} \ No newline at end of file diff --git a/docker-edge/docker-compose.yml b/docker-edge/docker-compose.yml index 8c6bc0026f..9b849b4de0 100644 --- a/docker-edge/docker-compose.yml +++ b/docker-edge/docker-compose.yml @@ -18,15 +18,16 @@ version: '3.0' services: - tb-edge: + tb-edge-1: restart: always image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:${TB_EDGE_VERSION}" ports: - "8082" - "1883" environment: - CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY}" - CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET}" + CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_1}" + CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_1}" + SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_1}" CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}" HTTP_BIND_PORT: "8082" CONF_FOLDER: "/config" @@ -34,8 +35,35 @@ services: - tb-edge.env volumes: - ./tb-edge/conf:/config - - ./tb-edge/data:/data - - ./tb-edge/log:/var/log/tb-edge + tb-edge-2: + restart: always + image: "${DOCKER_REPO}/${TB_EDGE_DOCKER_NAME}:${TB_EDGE_VERSION}" + ports: + - "8083" + - "1884" + environment: + CLOUD_ROUTING_KEY: "${CLOUD_ROUTING_KEY_2}" + CLOUD_ROUTING_SECRET: "${CLOUD_ROUTING_SECRET_2}" + SPRING_DATASOURCE_URL: "${SPRING_DATASOURCE_URL_2}" + CLOUD_RPC_HOST: "${CLOUD_RPC_HOST}" + HTTP_BIND_PORT: "8083" + CONF_FOLDER: "/config" + TB_QUEUE_TYPE: "kafka" + TB_KAFKA_SERVERS: "kafka-edge-2:9092" + env_file: + - tb-edge.env + volumes: + - ./tb-edge/conf:/config + kafka-edge-2: + restart: always + image: "bitnami/kafka:3.8.1" + ports: + - "9092" + environment: + KAFKA_CFG_ADVERTISED_LISTENERS: "INSIDE://:9094,OUTSIDE://kafka-edge-2:9092" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka-edge-2:9093" + env_file: + - kafka.env tb-monolith: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" @@ -56,6 +84,7 @@ services: LWM2M_ENABLED: "false" SNMP_ENABLED: "false" EDGES_ENABLED: "true" + #TB_QUEUE_TYPE: "kafka" TB_QUEUE_TYPE: "in-memory" TB_KAFKA_SERVERS: "kafka:9092" env_file: @@ -65,33 +94,14 @@ services: - ./tb-node/log:/var/log/thingsboard - ./tb-node/data:/data depends_on: - - zookeeper - kafka - zookeeper: - restart: always - image: "zookeeper:3.8.0" - ports: - - "2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - ZOO_ADMINSERVER_ENABLED: "false" kafka: restart: always - image: "bitnami/kafka:3.7.0" + image: "bitnami/kafka:3.8.1" ports: - "9092" environment: - ALLOW_PLAINTEXT_LISTENER: "yes" - KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_CFG_LISTENERS: "INSIDE://:9093,OUTSIDE://:9092" - KAFKA_CFG_ADVERTISED_LISTENERS: "INSIDE://:9093,OUTSIDE://kafka:9092" - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "INSIDE" - KAFKA_CFG_LOG_RETENTION_BYTES: "1073741824" - KAFKA_CFG_SEGMENT_BYTES: "268435456" - KAFKA_CFG_LOG_RETENTION_MS: "300000" - KAFKA_CFG_LOG_CLEANUP_POLICY: "delete" - depends_on: - - zookeeper + KAFKA_CFG_ADVERTISED_LISTENERS: "INSIDE://:9094,OUTSIDE://kafka:9092" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka:9093" + env_file: + - kafka.env diff --git a/docker-edge/kafka.env b/docker-edge/kafka.env new file mode 100644 index 0000000000..2da9c92808 --- /dev/null +++ b/docker-edge/kafka.env @@ -0,0 +1,15 @@ +ALLOW_PLAINTEXT_LISTENER=yes +KAFKA_CFG_LISTENERS=OUTSIDE://:9092,CONTROLLER://:9093,INSIDE://:9094 +KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT,CONTROLLER:PLAINTEXT +KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INSIDE +KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false +KAFKA_CFG_LOG_RETENTION_BYTES=1073741824 +KAFKA_CFG_SEGMENT_BYTES=268435456 +KAFKA_CFG_LOG_RETENTION_MS=300000 +KAFKA_CFG_LOG_CLEANUP_POLICY=delete +KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 +KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 +KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 +KAFKA_CFG_PROCESS_ROLES=controller,broker +KAFKA_CFG_NODE_ID=0 +KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \ No newline at end of file diff --git a/docker-edge/tb-edge.env b/docker-edge/tb-edge.env index 2d0efb73c0..d3699ccc93 100644 --- a/docker-edge/tb-edge.env +++ b/docker-edge/tb-edge.env @@ -1,5 +1,4 @@ SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver -SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/tb_edge SPRING_DATASOURCE_USERNAME=postgres SPRING_DATASOURCE_PASSWORD=postgres diff --git a/docker-edge/tb-node.env b/docker-edge/tb-node.env index 3f012070e4..b26ed0d8ef 100644 --- a/docker-edge/tb-node.env +++ b/docker-edge/tb-node.env @@ -1,4 +1,3 @@ -DATABASE_TS_TYPE=sql SPRING_JPA_DATABASE_PLATFORM=org.hibernate.dialect.PostgreSQLDialect SPRING_DRIVER_CLASS_NAME=org.postgresql.Driver SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/thingsboard diff --git a/msa/edge-black-box-tests/README.md b/msa/edge-black-box-tests/README.md index 8a79b049e7..f05863856d 100644 --- a/msa/edge-black-box-tests/README.md +++ b/msa/edge-black-box-tests/README.md @@ -1,3 +1,5 @@ +## Command to cleanup test log file +sed -E 's|^[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3} \[docker-java-stream-[^]]*\] INFO org\.testcontainers\.containers\.DockerComposeContainer -- ||g; s|/[A-Za-z0-9]{12}_||g; s| STDOUT: | |g' /tmp/edge-test.log > /tmp/edge-test-clean.log ## Edge Black box tests execution To run the black box tests with using Docker, the local Docker images of Thingsboard's microservices should be built.
diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index 639f3eb2cb..a3d8dd6c96 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -90,7 +90,6 @@ import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import java.io.IOException; import java.util.ArrayList; @@ -101,27 +100,48 @@ import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j public abstract class AbstractContainerTest { - public static final String CLOUD_ROUTING_KEY = "280629c7-f853-ee3d-01c0-fffbb6f2ef38"; - public static final String CLOUD_ROUTING_SECRET = "g9ta4soeylw6smqkky8g"; public static final String TB_MONOLITH_SERVICE_NAME = "tb-monolith"; public static final String TB_EDGE_SERVICE_NAME = "tb-edge"; - protected static final String CUSTOM_DEVICE_PROFILE_NAME = "Custom Device Profile"; + public static final List edgeConfigurations = + Arrays.asList( + new TestEdgeConfiguration("280629c7-f853-ee3d-01c0-fffbb6f2ef38", "g9ta4soeylw6smqkky8g", 8082, 1, "Edge-in-memory"), + new TestEdgeConfiguration("e29dadb1-c487-3b9e-1b5a-02193191c90e", "dmb17p71vz9svfl7tgnz", 8083, 2, "Edge-kafka")); + + protected static List testParameters = new ArrayList<>(); + protected static RestClient cloudRestClient = null; + protected static String tbUrl; + protected static Edge edge; + protected static String edgeUrl; protected static RestClient edgeRestClient; - protected static Edge edge; + protected void performTestOnEachEdge(Runnable runnable) { + for (TestEdgeRuntimeParameters edgeTestParameter : testParameters) { + edge = edgeTestParameter.getEdge(); + edgeUrl = edgeTestParameter.getUrl(); + edgeRestClient = edgeTestParameter.getRestClient(); - protected static EdgeVersion edgeVersion; + long startTime = System.currentTimeMillis(); + log.info("================================================="); + log.info("STARTING TEST: {} for edge {} {}", Thread.currentThread().getStackTrace()[2].getMethodName(), edge.getName(), edge.getRoutingKey()); + log.info("================================================="); - protected static String tbUrl; - protected static String edgeUrl; + runnable.run(); + + long elapsedTime = System.currentTimeMillis() - startTime; + log.info("================================================="); + log.info("SUCCEEDED TEST: {} for edge {} {} in {} ms", Thread.currentThread().getStackTrace()[2].getMethodName(), edge.getName(), edge.getRoutingKey(), elapsedTime); + log.info("================================================="); + } + } @BeforeClass public static void before() throws Exception { @@ -132,34 +152,42 @@ public static void before() throws Exception { cloudRestClient = new RestClient(tbUrl); cloudRestClient.login("tenant@thingsboard.org", "tenant"); - String edgeHost = ContainerTestSuite.testContainer.getServiceHost(TB_EDGE_SERVICE_NAME, 8082); - Integer edgePort = ContainerTestSuite.testContainer.getServicePort(TB_EDGE_SERVICE_NAME, 8082); - edgeUrl = "http://" + edgeHost + ":" + edgePort; - edgeRestClient = new RestClient(edgeUrl); - RuleChainId ruleChainId = updateRootRuleChain(); RuleChainId edgeRuleChainId = updateEdgeRootRuleChain(); - edge = createEdge("test", CLOUD_ROUTING_KEY, CLOUD_ROUTING_SECRET); + for (TestEdgeConfiguration config : edgeConfigurations) { + String edgeHost = ContainerTestSuite.testContainer.getServiceHost(TB_EDGE_SERVICE_NAME + "-" + config.getIdx(), config.getPort()); + Integer edgePort = ContainerTestSuite.testContainer.getServicePort(TB_EDGE_SERVICE_NAME + "-" + config.getIdx(), config.getPort()); + String edgeUrl = "http://" + edgeHost + ":" + edgePort; + Edge edge = createEdge(config.getName(), config.getRoutingKey(), config.getSecret()); + testParameters.add(new TestEdgeRuntimeParameters(new RestClient(edgeUrl), edge, edgeUrl)); + } - loginIntoEdgeWithRetries("tenant@thingsboard.org", "tenant"); + createCustomDeviceProfile(CUSTOM_DEVICE_PROFILE_NAME, ruleChainId, edgeRuleChainId); - getEdgeVersion(); + for (TestEdgeRuntimeParameters testParameter : testParameters) { + edgeRestClient = testParameter.getRestClient(); + edge = testParameter.getEdge(); + edgeUrl = testParameter.getUrl(); - Optional tenant = edgeRestClient.getTenantById(edge.getTenantId()); - Assert.assertTrue(tenant.isPresent()); - Assert.assertEquals(edge.getTenantId(), tenant.get().getId()); + log.info("================================================="); + log.info("STARTING INIT for edge {}", edge.getName()); + log.info("================================================="); - createCustomDeviceProfile(CUSTOM_DEVICE_PROFILE_NAME, ruleChainId, edgeRuleChainId); + loginIntoEdgeWithRetries("tenant@thingsboard.org", "tenant"); - // This is a starting point to start other tests - verifyWidgetBundles(); - } - } + Optional tenant = edgeRestClient.getTenantById(edge.getTenantId()); + Assert.assertTrue(tenant.isPresent()); + Assert.assertEquals(edge.getTenantId(), tenant.get().getId()); + + // This is a starting point to start other tests + verifyWidgetBundles(); - private static void getEdgeVersion() { - List attributes = cloudRestClient.getAttributeKvEntries(edge.getId(), List.of(DataConstants.EDGE_VERSION_ATTR_KEY)); - edgeVersion = EdgeVersion.valueOf(attributes.get(0).getValueAsString()); + log.info("================================================="); + log.info("SUCCEEDED INIT for edge {}", edge.getName()); + log.info("================================================="); + } + } } protected static void loginIntoEdgeWithRetries(String userName, String password) { @@ -184,38 +212,56 @@ protected static void loginIntoEdgeWithRetries(String userName, String password) private static void verifyWidgetBundles() { Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(30, TimeUnit.SECONDS). + .atMost(60, TimeUnit.SECONDS). until(() -> { try { long totalElements = edgeRestClient.getWidgetsBundles(new PageLink(100)).getTotalElements(); - final long expectedCount = 28; + final long expectedCount = 30; if (totalElements != expectedCount) { log.warn("Expected {} widget bundles, but got {}", expectedCount, totalElements); } return totalElements == expectedCount; } catch (Throwable e) { + log.error("Failed to verify widget bundles", e); return false; } }); PageData pageData = edgeRestClient.getWidgetsBundles(new PageLink(100)); - for (WidgetsBundleId widgetsBundleId : pageData.getData().stream().map(WidgetsBundle::getId).toList()) { + for (WidgetsBundle widgetsBundle : pageData.getData()) { Awaitility.await() .pollInterval(1000, TimeUnit.MILLISECONDS) - .atMost(60, TimeUnit.SECONDS). + .atMost(90, TimeUnit.SECONDS). until(() -> { try { - List edgeBundleWidgetTypes = edgeRestClient.getBundleWidgetTypes(widgetsBundleId); - List cloudBundleWidgetTypes = cloudRestClient.getBundleWidgetTypes(widgetsBundleId); - return cloudBundleWidgetTypes != null && edgeBundleWidgetTypes != null - && edgeBundleWidgetTypes.size() == cloudBundleWidgetTypes.size(); + List edgeBundleWidgetTypes = edgeRestClient.getBundleWidgetTypes(widgetsBundle.getId()); + List cloudBundleWidgetTypes = cloudRestClient.getBundleWidgetTypes(widgetsBundle.getId()); + if (cloudBundleWidgetTypes == null || edgeBundleWidgetTypes == null) { + return false; + } + if (edgeBundleWidgetTypes.size() != cloudBundleWidgetTypes.size()) { + // Collect the names of the widget types for edge and cloud + String cloudWidgetTypeNames = cloudBundleWidgetTypes.stream() + .map(WidgetType::getName) + .collect(Collectors.joining(", ")); + String edgeWidgetTypeNames = edgeBundleWidgetTypes.stream() + .map(WidgetType::getName) + .collect(Collectors.joining(", ")); + log.warn("Expected {} widget types, but got {}. " + + "widgetBundleId = {}, widgetsBundleName = {}" + + "cloudBundleWidgetTypesNames = [{}], edgeBundleWidgetTypesNames = [{}]", + cloudBundleWidgetTypes.size(), edgeBundleWidgetTypes.size(), + widgetsBundle.getId(), widgetsBundle.getName(), + cloudWidgetTypeNames, edgeWidgetTypeNames); + } + return edgeBundleWidgetTypes.size() == cloudBundleWidgetTypes.size(); } catch (Throwable e) { return false; } }); - List edgeBundleWidgetTypes = edgeRestClient.getBundleWidgetTypes(widgetsBundleId); - List cloudBundleWidgetTypes = cloudRestClient.getBundleWidgetTypes(widgetsBundleId); + List edgeBundleWidgetTypes = edgeRestClient.getBundleWidgetTypes(widgetsBundle.getId()); + List cloudBundleWidgetTypes = cloudRestClient.getBundleWidgetTypes(widgetsBundle.getId()); Assert.assertNotNull("edgeBundleWidgetTypes can't be null", edgeBundleWidgetTypes); Assert.assertNotNull("cloudBundleWidgetTypes can't be null", cloudBundleWidgetTypes); } @@ -300,10 +346,6 @@ protected static DeviceProfile createDeviceProfileOnEdge(String name) { return doCreateDeviceProfile(name, null, null, new DefaultDeviceProfileTransportConfiguration(), edgeRestClient); } - protected static DeviceProfile createDeviceProfileOnEdge(String name, RuleChainId defaultRuleChain, RuleChainId defaultEdgeRuleChainId) { - return doCreateDeviceProfile(name, defaultRuleChain, defaultEdgeRuleChainId, new DefaultDeviceProfileTransportConfiguration(), edgeRestClient); - } - private static DeviceProfile doCreateDeviceProfile(String name, RuleChainId defaultRuleChain, RuleChainId defaultEdgeRuleChainId, DeviceProfileTransportConfiguration deviceProfileTransportConfiguration, RestClient restClient) { DeviceProfile deviceProfile = new DeviceProfile(); @@ -358,7 +400,7 @@ protected static void extendDeviceProfileData(DeviceProfile deviceProfile) { protected static Edge createEdge(String name, String routingKey, String secret) { Edge edge = new Edge(); - edge.setName(name + StringUtils.randomAlphanumeric(7)); + edge.setName(name); edge.setType("DEFAULT"); edge.setRoutingKey(routingKey); edge.setSecret(secret); @@ -827,7 +869,7 @@ protected void unAssignFromEdgeAndDeleteDashboard(DashboardId dashboardId) { cloudRestClient.deleteDashboard(dashboardId); } - protected OtaPackageId createOtaPackageInfo(DeviceProfileId deviceProfileId, OtaPackageType otaPackageType) throws Exception { + protected OtaPackageId createOtaPackageInfo(DeviceProfileId deviceProfileId, OtaPackageType otaPackageType) { OtaPackageInfo otaPackageInfo = new OtaPackageInfo(); otaPackageInfo.setDeviceProfileId(deviceProfileId); otaPackageInfo.setType(otaPackageType); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c9f0d3a66c..000a3337d3 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -28,29 +28,30 @@ import java.util.HashMap; import java.util.UUID; -import static org.thingsboard.server.msa.AbstractContainerTest.CLOUD_ROUTING_KEY; -import static org.thingsboard.server.msa.AbstractContainerTest.CLOUD_ROUTING_SECRET; import static org.thingsboard.server.msa.AbstractContainerTest.TB_EDGE_SERVICE_NAME; import static org.thingsboard.server.msa.AbstractContainerTest.TB_MONOLITH_SERVICE_NAME; +import static org.thingsboard.server.msa.AbstractContainerTest.edgeConfigurations; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.edge.*Test"}) @Slf4j public class ContainerTestSuite { - public static DockerComposeContainer testContainer; - private static final String SOURCE_DIR = "./../../docker-edge/"; + public static DockerComposeContainer testContainer; + @ClassRule public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller(); @ClassRule public static DockerComposeContainer getTestContainer() { HashMap env = new HashMap<>(); - env.put("CLOUD_ROUTING_KEY", CLOUD_ROUTING_KEY); - env.put("CLOUD_ROUTING_SECRET", CLOUD_ROUTING_SECRET); env.put("CLOUD_RPC_HOST", TB_MONOLITH_SERVICE_NAME); + for (TestEdgeConfiguration config : edgeConfigurations) { + env.put("CLOUD_ROUTING_KEY_" + config.getIdx(), config.getRoutingKey()); + env.put("CLOUD_ROUTING_SECRET_" + config.getIdx(), config.getSecret()); + } if (testContainer == null) { try { @@ -84,9 +85,14 @@ public void stop() { .withEnv(installTb.getEnv()) .withEnv(env) .withExposedService(TB_MONOLITH_SERVICE_NAME, 8080) - .withExposedService(TB_EDGE_SERVICE_NAME, 8082) - .withExposedService("zookeeper", 2181) .withExposedService("kafka", 9092); + for (TestEdgeConfiguration edgeConfiguration : edgeConfigurations) { + testContainer.withExposedService(TB_EDGE_SERVICE_NAME + "-" + edgeConfiguration.getIdx(), edgeConfiguration.getPort()); + if (edgeConfiguration.getName().contains("kafka")) { + testContainer.withExposedService("kafka-edge-" + edgeConfiguration.getIdx(), 9092); + } + } + } catch (Exception e) { log.error("Failed to create test container", e); Assert.fail("Failed to create test container"); @@ -103,5 +109,4 @@ private static void tryDeleteDir(String targetDir) { log.error("Can't delete temp directory " + targetDir, e); } } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/UplinkMessageService.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeConfiguration.java similarity index 65% rename from application/src/main/java/org/thingsboard/server/service/cloud/UplinkMessageService.java rename to msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeConfiguration.java index b2dc3e0d15..ed441e5ce3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/UplinkMessageService.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeConfiguration.java @@ -13,15 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cloud; +package org.thingsboard.server.msa; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; - -public interface UplinkMessageService { - - void processHandleMessages(TenantId tenantId) throws Exception; - - void onUplinkResponse(UplinkResponseMsg msg); +import lombok.AllArgsConstructor; +import lombok.Data; +@AllArgsConstructor +@Data +public class TestEdgeConfiguration { + private String routingKey; + private String secret; + private Integer port; + private Integer idx; + private String name; } diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeRuntimeParameters.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeRuntimeParameters.java new file mode 100644 index 0000000000..9be616aa69 --- /dev/null +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/TestEdgeRuntimeParameters.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.rest.client.RestClient; +import org.thingsboard.server.common.data.edge.Edge; + +@AllArgsConstructor +@Data +public class TestEdgeRuntimeParameters { + private RestClient restClient; + private Edge edge; + private String url; +} diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 55fcfa4ea9..14de7ef4da 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -17,6 +17,7 @@ import io.github.cdimascio.dotenv.Dotenv; import io.github.cdimascio.dotenv.DotenvEntry; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; @@ -42,6 +43,8 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbLogVolume; private final String tbEdgeLogVolume; private final String tbEdgeDataVolume; + + @Getter private final Map env; public ThingsBoardDbInstaller() { @@ -68,8 +71,11 @@ public ThingsBoardDbInstaller() { } env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); env.put("TB_LOG_VOLUME", tbLogVolume); - env.put("TB_EDGE_LOG_VOLUME", tbEdgeLogVolume); - env.put("TB_EDGE_DATA_VOLUME", tbEdgeDataVolume); + for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) { + env.put("SPRING_DATASOURCE_URL_" + edgeEnv, "jdbc:postgresql://postgres:5432/tb_edge_" + edgeEnv); + env.put("TB_EDGE_LOG_VOLUME_" + edgeEnv, tbEdgeLogVolume + "-" + edgeEnv); + env.put("TB_EDGE_DATA_VOLUME_" + edgeEnv, tbEdgeDataVolume + "-" + edgeEnv); + } dockerCompose.withEnv(env); } catch (Exception e) { @@ -78,10 +84,6 @@ public ThingsBoardDbInstaller() { } } - public Map getEnv() { - return env; - } - @Override protected void before() throws Throwable { try { @@ -91,12 +93,13 @@ protected void before() throws Throwable { dockerCompose.withCommand("volume create " + tbLogVolume); dockerCompose.invokeDocker(); + for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) { + dockerCompose.withCommand("volume create " + tbEdgeLogVolume + "-" + edgeEnv); + dockerCompose.invokeDocker(); - dockerCompose.withCommand("volume create " + tbEdgeLogVolume); - dockerCompose.invokeDocker(); - - dockerCompose.withCommand("volume create " + tbEdgeDataVolume); - dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbEdgeDataVolume + "-" + edgeEnv); + dockerCompose.invokeDocker(); + } dockerCompose.withCommand("up -d postgres"); dockerCompose.invokeCompose(); @@ -104,15 +107,16 @@ protected void before() throws Throwable { dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-monolith"); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB_EDGE=true -e LOAD_DEMO=true tb-edge"); - dockerCompose.invokeCompose(); - + for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) { + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB_EDGE=true -e LOAD_DEMO=true tb-edge" + "-" + edgeEnv); + dockerCompose.invokeCompose(); + } dockerCompose.withCommand("exec -T postgres psql -U postgres -d thingsboard -f /custom-sql/thingsboard.sql"); dockerCompose.invokeCompose(); - - dockerCompose.withCommand("exec -T postgres psql -U postgres -d tb_edge -f /custom-sql/tb_edge.sql"); - dockerCompose.invokeCompose(); - + for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) { + dockerCompose.withCommand("exec -T postgres psql -U postgres -d tb_edge" + "_" + edgeEnv + " -f /custom-sql/tb_edge.sql"); + dockerCompose.invokeCompose(); + } } finally { try { dockerCompose.withCommand("down -v"); @@ -126,11 +130,13 @@ protected void before() throws Throwable { @Override protected void after() { try { - copyLogs(tbLogVolume, "./target/tb-logs/"); - copyLogs(tbEdgeLogVolume, "./target/tb-edge-logs/"); + for (int edgeEnv = 1; edgeEnv <= 2; edgeEnv++) { + copyLogs(tbLogVolume, "./target/tb-logs/"); + copyLogs(tbEdgeLogVolume + "-" + edgeEnv, "./target/tb-edge-logs/"); - dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbEdgeLogVolume); - dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbEdgeLogVolume + "-" + edgeEnv); + dockerCompose.invokeDocker(); + } } catch (Exception e) { log.error("Failed [after]", e); throw e; @@ -147,7 +153,7 @@ private void copyLogs(String volumeName, String targetDir) { dockerCompose.withCommand("run -d --rm --name " + logsContainerName + " -v " + volumeName + ":/root alpine tail -f /dev/null"); dockerCompose.invokeDocker(); - dockerCompose.withCommand("cp " + logsContainerName + ":/root/. "+tbLogsDir.getAbsolutePath()); + dockerCompose.withCommand("cp " + logsContainerName + ":/root/. " + tbLogsDir.getAbsolutePath()); dockerCompose.invokeDocker(); dockerCompose.withCommand("rm -f " + logsContainerName); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AlarmClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AlarmClientTest.java index 42c1b773a2..4d5d114be1 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AlarmClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AlarmClientTest.java @@ -56,6 +56,10 @@ public class AlarmClientTest extends AbstractContainerTest { @Test public void testAlarms() { + performTestOnEachEdge(this::_testAlarms); + } + + private void _testAlarms() { cloudRemoveFromSaveTimeseriesToPushToNodeSuccessConnection(); // create alarm @@ -135,6 +139,10 @@ private Optional getLatestAnyAlarmByEntityId(EntityId entityId, RestC @Test public void sendAlarmToCloud() { + performTestOnEachEdge(this::_sendAlarmToCloud); + } + + private void _sendAlarmToCloud() { edgeRemoveFromSaveTimeseriesToPushToNodeSuccessConnection(); // create alarm on edge @@ -203,6 +211,10 @@ private Optional getLatestAlarmByEntityIdFromCloud(EntityId entityId) @Test public void testAlarmComments() { + performTestOnEachEdge(this::_testAlarmComments); + } + + private void _testAlarmComments() { cloudRemoveFromSaveTimeseriesToPushToNodeSuccessConnection(); // create alarm @@ -273,6 +285,10 @@ public void testAlarmComments() { @Test public void sendAlarmCommentToCloud() { + performTestOnEachEdge(this::_sendAlarmCommentToCloud); + } + + private void _sendAlarmCommentToCloud() { edgeRemoveFromSaveTimeseriesToPushToNodeSuccessConnection(); // create alarm diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetClientTest.java index 9699eaaa81..eac58b5857 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetClientTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; -import org.junit.Ignore; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; @@ -37,6 +36,10 @@ public class AssetClientTest extends AbstractContainerTest { @Test public void testAssets() { + performTestOnEachEdge(this::_testAssets); + } + + private void _testAssets() { // create asset #1 and assign to edge Asset savedAsset1 = saveAndAssignAssetToEdge("Building"); cloudRestClient.assignAssetToEdge(edge.getId(), savedAsset1.getId()); @@ -115,6 +118,10 @@ public void testAssets() { @Test public void testSendAssetToCloud() { + performTestOnEachEdge(this::_testSendAssetToCloud); + } + + private void _testSendAssetToCloud() { // create asset on edge String defaultAssetProfileName = edgeRestClient.getDefaultAssetProfileInfo().getName(); Asset savedAssetOnEdge = saveAssetOnEdge("Edge Asset 2", defaultAssetProfileName); @@ -170,6 +177,10 @@ public void testSendAssetToCloud() { @Test public void testSendAssetToCloudWithNameThatAlreadyExistsOnCloud() { + performTestOnEachEdge(this::_testSendAssetToCloudWithNameThatAlreadyExistsOnCloud); + } + + private void _testSendAssetToCloudWithNameThatAlreadyExistsOnCloud() { // create asset on cloud and edge with the same name Asset savedAssetOnCloud = saveAssetOnCloud("Edge Asset 3", "Building"); Awaitility.await() diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetProfileClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetProfileClientTest.java index 22dd6d7180..0abb2b63c6 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetProfileClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/AssetProfileClientTest.java @@ -35,6 +35,10 @@ public class AssetProfileClientTest extends AbstractContainerTest { @Test public void testAssetProfiles() { + performTestOnEachEdge(this::_testAssetProfiles); + } + + private void _testAssetProfiles() { verifyAssetProfilesOnEdge(1); // create asset profile @@ -61,20 +65,25 @@ public void testAssetProfiles() { @Test public void testAssetProfileToCloud() { + performTestOnEachEdge(this::_testAssetProfileToCloud); + } + + private void _testAssetProfileToCloud() { // create asset profile on edge - AssetProfile saveAssetProfileOnEdge = saveAssetProfileOnEdge("Asset Profile On Edge"); + AssetProfile saveAssetProfileOnEdge = saveAssetProfileOnEdge("Asset Profile On Edge " + edge.getName()); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) .until(() -> cloudRestClient.getAssetProfileById(saveAssetProfileOnEdge.getId()).isPresent()); // update asset profile - saveAssetProfileOnEdge.setName("Asset Profile On Edge Updated"); + String updatedAssetProfileName = "Asset Profile On Edge Updated " + edge.getName(); + saveAssetProfileOnEdge.setName(updatedAssetProfileName); edgeRestClient.saveAssetProfile(saveAssetProfileOnEdge); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) - .until(() -> "Asset Profile On Edge Updated".equals(cloudRestClient.getAssetProfileById(saveAssetProfileOnEdge.getId()).get().getName())); + .until(() -> (updatedAssetProfileName).equals(cloudRestClient.getAssetProfileById(saveAssetProfileOnEdge.getId()).get().getName())); // cleanup - we can delete asset profile only on Cloud cloudRestClient.deleteAssetProfile(saveAssetProfileOnEdge.getId()); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/CustomerClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/CustomerClientTest.java index 5cc2436f2a..dbdcd1ab52 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/CustomerClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/CustomerClientTest.java @@ -28,6 +28,10 @@ public class CustomerClientTest extends AbstractContainerTest { @Test public void testCreateUpdateDeleteCustomer() { + performTestOnEachEdge(this::_testCreateUpdateDeleteCustomer); + } + + private void _testCreateUpdateDeleteCustomer() { // create customer Customer customer = new Customer(); customer.setTitle("Test Customer"); @@ -53,6 +57,10 @@ public void testCreateUpdateDeleteCustomer() { @Test public void testPublicCustomerCreatedOnEdge() { + performTestOnEachEdge(this::_testPublicCustomerCreatedOnEdge); + } + + private void _testPublicCustomerCreatedOnEdge() { Customer publicCustomer = findPublicCustomer(); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DashboardClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DashboardClientTest.java index 25b71c593d..a0223853da 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DashboardClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DashboardClientTest.java @@ -33,6 +33,10 @@ public class DashboardClientTest extends AbstractContainerTest { @Test public void testDashboards() { + performTestOnEachEdge(this::_testDashboards); + } + + private void _testDashboards() { // create dashboard #1 and assign to edge Dashboard savedDashboard1 = saveDashboardOnCloud("Edge Dashboard 1"); cloudRestClient.assignDashboardToEdge(edge.getId(), savedDashboard1.getId()); @@ -109,6 +113,10 @@ public void testDashboards() { @Test public void testSendDashboardToCloud() { + performTestOnEachEdge(this::_testSendDashboardToCloud); + } + + private void _testSendDashboardToCloud() { // create dashboard on edge Dashboard savedDashboardOnEdge = saveDashboardOnEdge("Edge Dashboard 3"); Awaitility.await() diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceClientTest.java index 6183d46ede..2f3da5b6b3 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceClientTest.java @@ -72,7 +72,11 @@ public class DeviceClientTest extends AbstractContainerTest { @Test - public void testDevices() throws Exception { + public void testDevices() { + performTestOnEachEdge(this::_testDevices); + } + + private void _testDevices() { String deviceProfileName = "Remote Controller"; // create device #1 and assign to edge Device savedDevice1 = saveAndAssignDeviceToEdge(deviceProfileName); @@ -186,6 +190,10 @@ public void testDevices() throws Exception { @Test public void sendDeviceToCloud() { + performTestOnEachEdge(this::_sendDeviceToCloud); + } + + private void _sendDeviceToCloud() { // create device on edge Device savedDeviceOnEdge = saveDeviceOnEdge("Edge Device 2", "default"); DeviceId savedDeviceOnEdgeId = savedDeviceOnEdge.getId(); @@ -389,6 +397,10 @@ private void verifyDeviceCredentialsOnCloudAndEdge(Device savedDevice) { @Test public void testProvisionDevice() { + performTestOnEachEdge(this::_testProvisionDevice); + } + + private void _testProvisionDevice() { final String DEVICE_PROFILE_NAME = "Provision Device Profile"; final String DEVICE_NAME = "Provisioned Device"; @@ -476,6 +488,10 @@ private Device getDeviceByNameAndType(String deviceName, String type, RestClient @Test public void testOneWayRpcCall() { + performTestOnEachEdge(this::_testOneWayRpcCall); + } + + private void _testOneWayRpcCall() { // create device on cloud and assign to edge Device device = saveAndAssignDeviceToEdge(); @@ -530,6 +546,10 @@ public void testOneWayRpcCall() { @Test public void testTwoWayRpcCall() { + performTestOnEachEdge(this::_testTwoWayRpcCall); + } + + private void _testTwoWayRpcCall() { // create device on cloud and assign to edge Device device = saveAndAssignDeviceToEdge(); @@ -607,6 +627,10 @@ public void testTwoWayRpcCall() { @Test public void testClientRpcCallToCloud() { + performTestOnEachEdge(this::_testClientRpcCallToCloud); + } + + private void _testClientRpcCallToCloud() { // create device on cloud and assign to edge Device device = saveAndAssignDeviceToEdge(); @@ -642,6 +666,10 @@ public void testClientRpcCallToCloud() { @Test public void sendDeviceWithNameThatAlreadyExistsOnCloud() { + performTestOnEachEdge(this::_sendDeviceWithNameThatAlreadyExistsOnCloud); + } + + private void _sendDeviceWithNameThatAlreadyExistsOnCloud() { String deviceName = StringUtils.randomAlphanumeric(15); Device savedDeviceOnCloud = saveDeviceOnCloud(deviceName, "default"); Device savedDeviceOnEdge = saveDeviceOnEdge(deviceName, "default"); @@ -664,6 +692,10 @@ public void sendDeviceWithNameThatAlreadyExistsOnCloud() { @Test public void testClaimDevice() { + performTestOnEachEdge(this::_testClaimDevice); + } + + private void _testClaimDevice() { // create customer, user and device Customer customer = new Customer(); customer.setTitle("Claim Test Customer"); @@ -714,6 +746,10 @@ public void testClaimDevice() { @Test public void testSharedAttributeUpdates() { + performTestOnEachEdge(this::_testSharedAttributeUpdates); + } + + private void _testSharedAttributeUpdates() { // create device on cloud and assign to edge Device savedDevice = saveAndAssignDeviceToEdge(); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceProfileClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceProfileClientTest.java index 944080a8d9..ed3ed4fb2f 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceProfileClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/DeviceProfileClientTest.java @@ -143,7 +143,11 @@ public class DeviceProfileClientTest extends AbstractContainerTest { " }"; @Test - public void testDeviceProfiles() throws Exception { + public void testDeviceProfiles() { + performTestOnEachEdge(this::_testDeviceProfiles); + } + + private void _testDeviceProfiles() { verifyDeviceProfilesOnEdge(3); // create device profile @@ -207,20 +211,25 @@ public void testDeviceProfiles() throws Exception { @Test public void testDeviceProfileToCloud() { + performTestOnEachEdge(this::_testDeviceProfileToCloud); + } + + private void _testDeviceProfileToCloud() { // create device profile on edge - DeviceProfile saveDeviceProfileOnEdge = createDeviceProfileOnEdge("Device Profile On Edge"); + DeviceProfile saveDeviceProfileOnEdge = createDeviceProfileOnEdge("Device Profile On Edge " + edge.getName()); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) .until(() -> cloudRestClient.getDeviceProfileById(saveDeviceProfileOnEdge.getId()).isPresent()); // update asset profile - saveDeviceProfileOnEdge.setName("Device Profile On Edge Updated"); + String updatedDeviceProfileName = "Device Profile On Edge Updated " + edge.getName(); + saveDeviceProfileOnEdge.setName(updatedDeviceProfileName); edgeRestClient.saveDeviceProfile(saveDeviceProfileOnEdge); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) - .until(() -> "Device Profile On Edge Updated".equals(cloudRestClient.getDeviceProfileById(saveDeviceProfileOnEdge.getId()).get().getName())); + .until(() -> updatedDeviceProfileName.equals(cloudRestClient.getDeviceProfileById(saveDeviceProfileOnEdge.getId()).get().getName())); // cleanup - we can delete asset profile only on Cloud cloudRestClient.deleteDeviceProfile(saveDeviceProfileOnEdge.getId()); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EdgeClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EdgeClientTest.java index 6a5ea6f44e..beac5dd8d3 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EdgeClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EdgeClientTest.java @@ -16,19 +16,19 @@ package org.thingsboard.server.msa.edge; import lombok.extern.slf4j.Slf4j; -import org.awaitility.Awaitility; import org.junit.Test; import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.msa.AbstractContainerTest; -import java.util.concurrent.TimeUnit; - @Slf4j public class EdgeClientTest extends AbstractContainerTest { @Test public void testEdge_assignToCustomer_unassignFromCustomer() { + performTestOnEachEdge(this::_testEdge_assignToCustomer_unassignFromCustomer); + } + + private void _testEdge_assignToCustomer_unassignFromCustomer() { // assign edge to customer Customer customer = new Customer(); customer.setTitle("Edge Test Customer"); @@ -39,6 +39,9 @@ public void testEdge_assignToCustomer_unassignFromCustomer() { // unassign edge from customer unassignEdgeFromCustomerAndValidateUnassignmentOnCloud(); + + // cleanup + cloudRestClient.deleteCustomer(savedCustomer.getId()); } } diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EntityViewClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EntityViewClientTest.java index 9ea156752d..78f92f104c 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EntityViewClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/EntityViewClientTest.java @@ -36,6 +36,10 @@ public class EntityViewClientTest extends AbstractContainerTest { @Test public void testEntityViews() { + performTestOnEachEdge(this::_testEntityViews); + } + + private void _testEntityViews() { // create entity view #1 and assign to edge Device device = saveAndAssignDeviceToEdge(); EntityView savedEntityView1 = saveEntityViewOnCloud("Edge Entity View 1", "Default", device.getId()); @@ -102,6 +106,10 @@ public void testEntityViews() { @Test public void testSendEntityViewToCloud() { + performTestOnEachEdge(this::_testSendEntityViewToCloud); + } + + private void _testSendEntityViewToCloud() { // create asset on edge Asset savedAssetOnEdge = saveAssetOnEdge("Edge Asset For Entity View", edgeRestClient.getDefaultAssetProfileInfo().getName()); Awaitility.await() @@ -170,6 +178,10 @@ public void testSendEntityViewToCloud() { @Test public void testSendEntityViewToCloudWithNameThatAlreadyExistsOnCloud() { + performTestOnEachEdge(this::_testSendEntityViewToCloudWithNameThatAlreadyExistsOnCloud); + } + + private void _testSendEntityViewToCloudWithNameThatAlreadyExistsOnCloud() { // create entity view on cloud and edge with the same name Device device = saveAndAssignDeviceToEdge(); EntityView savedEntityViewOnCloud = saveEntityViewOnCloud("Edge Entity View Exists", "Default", device.getId()); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/OtaPackageClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/OtaPackageClientTest.java index bebac2f7b0..50c98f671f 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/OtaPackageClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/OtaPackageClientTest.java @@ -38,7 +38,11 @@ public class OtaPackageClientTest extends AbstractContainerTest { @Test - public void testOtaPackages() throws Exception { + public void testOtaPackages() { + performTestOnEachEdge(this::_testOtaPackages); + } + + private void _testOtaPackages() { // create ota package DeviceProfileInfo defaultDeviceProfileInfo = cloudRestClient.getDefaultDeviceProfileInfo(); OtaPackageId otaPackageId = createOtaPackageInfo(new DeviceProfileId(defaultDeviceProfileInfo.getId().getId()), FIRMWARE); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/QueueClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/QueueClientTest.java index b908e1b87e..44ca382fca 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/QueueClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/QueueClientTest.java @@ -37,6 +37,10 @@ public class QueueClientTest extends AbstractContainerTest { @Test public void testQueues() { + performTestOnEachEdge(this::_testQueues); + } + + private void _testQueues() { cloudRestClient.login("sysadmin@thingsboard.org", "sysadmin"); // create queue diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RelationClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RelationClientTest.java index d9563fd8e3..063ea5d2d2 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RelationClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RelationClientTest.java @@ -31,6 +31,10 @@ public class RelationClientTest extends AbstractContainerTest { @Test public void testRelations() { + performTestOnEachEdge(this::_testRelations); + } + + private void _testRelations() { // create relation Device device = saveAndAssignDeviceToEdge(); Asset asset = saveAndAssignAssetToEdge(); @@ -62,6 +66,10 @@ public void testRelations() { @Test public void sendRelationToCloud() { + performTestOnEachEdge(this::_sendRelationToCloud); + } + + private void _sendRelationToCloud() { Device device = saveAndAssignDeviceToEdge(); Device savedDeviceOnEdge = saveDeviceOnEdge("Test Device 3", "default"); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/ResourceClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/ResourceClientTest.java index b22655c844..47823698b3 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/ResourceClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/ResourceClientTest.java @@ -31,6 +31,10 @@ public class ResourceClientTest extends AbstractContainerTest { @Test public void testSendResourceToEdge() { + performTestOnEachEdge(this::_testSendResourceToEdge); + } + + private void _testSendResourceToEdge() { // create resource on cloud String title = "Resource on Cloud"; TbResource resource = saveResourceOnEdge(title, "ResourceCloud.js", cloudRestClient); @@ -62,6 +66,10 @@ public void testSendResourceToEdge() { @Test public void testSendResourceToCloud() { + performTestOnEachEdge(this::_testSendResourceToCloud); + } + + private void _testSendResourceToCloud() { // create resource on edge String title = "Resource on Edge"; TbResource resource = saveResourceOnEdge(title, "ResourceEdge.js", edgeRestClient); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RuleChainClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RuleChainClientTest.java index 9a24ac93fb..cda090e9a2 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RuleChainClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/RuleChainClientTest.java @@ -38,6 +38,10 @@ public class RuleChainClientTest extends AbstractContainerTest { @Test public void testRuleChains() { + performTestOnEachEdge(this::_testRuleChains); + } + + private void _testRuleChains() { Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) @@ -66,6 +70,10 @@ public void testRuleChains() { @Test public void testUpdateRootRuleChain() { + performTestOnEachEdge(this::_testUpdateRootRuleChain); + } + + private void _testUpdateRootRuleChain() { PageData ruleChains = cloudRestClient.getEdgeRuleChains(edge.getId(), new TimePageLink(100)); RuleChain rootRuleChain = ruleChains.getData().stream().filter(ruleChain -> ruleChain.getId().equals(edge.getRootRuleChainId())).findAny().orElse(null); Assert.assertNotNull(rootRuleChain); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TelemetryClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TelemetryClientTest.java index b6be082df7..8725641fe2 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TelemetryClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TelemetryClientTest.java @@ -40,6 +40,10 @@ public class TelemetryClientTest extends AbstractContainerTest { @Test public void testSendPostTelemetryRequestToCloud_performanceTest() { + performTestOnEachEdge(this::_testSendPostTelemetryRequestToCloud_performanceTest); + } + + private void _testSendPostTelemetryRequestToCloud_performanceTest() { Device device = saveAndAssignDeviceToEdge(); Awaitility.await() @@ -91,6 +95,10 @@ public void testSendPostTelemetryRequestToCloud_performanceTest() { @Test public void testSendPostTelemetryRequestToCloud() { + performTestOnEachEdge(this::_testSendPostTelemetryRequestToCloud); + } + + private void _testSendPostTelemetryRequestToCloud() { List keys = Arrays.asList("strTelemetryToCloud", "boolTelemetryToCloud", "doubleTelemetryToCloud", "longTelemetryToCloud"); JsonObject timeseriesPayload = new JsonObject(); @@ -119,6 +127,10 @@ public void testSendPostTelemetryRequestToCloud() { @Test public void testSendPostTelemetryRequestToEdge() { + performTestOnEachEdge(this::_testSendPostTelemetryRequestToEdge); + } + + private void _testSendPostTelemetryRequestToEdge() { List keys = Arrays.asList("strTelemetryToEdge", "boolTelemetryToEdge", "doubleTelemetryToEdge", "longTelemetryToEdge"); JsonObject timeseriesPayload = new JsonObject(); @@ -188,6 +200,10 @@ private List sendPostTelemetryRequest(RestClient sourceRestClient, St @Test public void testSendPostAttributesRequestToCloud() { + performTestOnEachEdge(this::_testSendPostAttributesRequestToCloud); + } + + private void _testSendPostAttributesRequestToCloud() { List keys = Arrays.asList("strAttrToCloud", "boolAttrToCloud", "doubleAttrToCloud", "longAttrToCloud"); JsonObject attrPayload = new JsonObject(); @@ -217,6 +233,10 @@ public void testSendPostAttributesRequestToCloud() { @Test public void testSendPostAttributesRequestToEdge() { + performTestOnEachEdge(this::_testSendPostAttributesRequestToEdge); + } + + private void _testSendPostAttributesRequestToEdge() { List keys = Arrays.asList("strAttrToEdge", "boolAttrToEdge", "doubleAttrToEdge", "longAttrToEdge"); JsonObject attrPayload = new JsonObject(); @@ -285,6 +305,10 @@ private List testSendPostAttributesRequest(RestClient sourceRe @Test public void testSendAttributesUpdatedToEdge() { + performTestOnEachEdge(this::_testSendAttributesUpdatedToEdge); + } + + private void _testSendAttributesUpdatedToEdge() { List keys = Arrays.asList("strAttrToEdge", "boolAttrToEdge", "doubleAttrToEdge", "longAttrToEdge"); JsonObject attrPayload = new JsonObject(); @@ -319,6 +343,10 @@ private void verifyAttributesUpdatedToEdge(List kvEntries) { @Test public void testSendAttributesUpdatedToCloud() { + performTestOnEachEdge(this::_testSendAttributesUpdatedToCloud); + } + + private void _testSendAttributesUpdatedToCloud() { List keys = Arrays.asList("strAttrToCloud", "boolAttrToCloud", "doubleAttrToCloud", "longAttrToCloud"); JsonObject attrPayload = new JsonObject(); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TenantClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TenantClientTest.java index 6207ce2da7..affa6cb809 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TenantClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/TenantClientTest.java @@ -15,40 +15,49 @@ */ package org.thingsboard.server.msa.edge; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.junit.Test; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.msa.AbstractContainerTest; +import java.sql.Time; import java.util.concurrent.TimeUnit; @Slf4j public class TenantClientTest extends AbstractContainerTest { - @SneakyThrows @Test public void testUpdateTenant() { + performTestOnEachEdge(this::_testUpdateTenant); + } + + private void _testUpdateTenant() { cloudRestClient.login("sysadmin@thingsboard.org", "sysadmin"); Tenant tenant = edgeRestClient.getTenantById(edge.getTenantId()).get(); + String originalCountry = tenant.getCountry(); + // update tenant - tenant.setCountry("Edge Update country: Ukraine"); + String updatedCountry = "Edge Update country: Ukraine"; + tenant.setCountry(updatedCountry); cloudRestClient.saveTenant(tenant); Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) - .until(() -> "Edge Update country: Ukraine".equals(edgeRestClient.getTenantById(tenant.getId()).get().getCountry())); + .until(() -> updatedCountry.equals(edgeRestClient.getTenantById(tenant.getId()).get().getCountry())); // create new tenant profile TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName("New Tenant Profile"); TenantProfile saveTenantProfile = cloudRestClient.saveTenantProfile(tenantProfile); + TenantProfileId originalTenantProfileId = tenant.getTenantProfileId(); + // update tenant with new tenant profile tenant.setTenantProfileId(saveTenantProfile.getId()); cloudRestClient.saveTenant(tenant); @@ -58,6 +67,17 @@ public void testUpdateTenant() { .atMost(30, TimeUnit.SECONDS) .until(() -> saveTenantProfile.getId().equals(edgeRestClient.getTenantById(tenant.getId()).get().getTenantProfileId())); + // cleanup + tenant.setCountry(originalCountry); + tenant.setTenantProfileId(originalTenantProfileId); + cloudRestClient.saveTenant(tenant); + cloudRestClient.deleteTenantProfile(saveTenantProfile.getId()); + + Awaitility.await() + .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .until(() -> originalTenantProfileId.equals(edgeRestClient.getTenantById(tenant.getId()).get().getTenantProfileId())); + cloudRestClient.login("tenant@thingsboard.org", "tenant"); } diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/UserClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/UserClientTest.java index 1d137674f8..4782352482 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/UserClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/UserClientTest.java @@ -31,6 +31,10 @@ public class UserClientTest extends AbstractContainerTest { @Test public void testCreateUpdateDeleteTenantUser() { + performTestOnEachEdge(this::_testCreateUpdateDeleteTenantUser); + } + + private void _testCreateUpdateDeleteTenantUser() { // create user User user = new User(); user.setAuthority(Authority.TENANT_ADMIN); @@ -69,6 +73,10 @@ public void testCreateUpdateDeleteTenantUser() { @Test public void testCreateUpdateDeleteCustomerUser() { + performTestOnEachEdge(this::_testCreateUpdateDeleteCustomerUser); + } + + private void _testCreateUpdateDeleteCustomerUser() { // create customer Customer customer = new Customer(); customer.setTitle("User Test Customer"); diff --git a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/WidgetBundleAndTypeClientTest.java b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/WidgetBundleAndTypeClientTest.java index bac92b0817..004a1e57e6 100644 --- a/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/WidgetBundleAndTypeClientTest.java +++ b/msa/edge-black-box-tests/src/test/java/org/thingsboard/server/msa/edge/WidgetBundleAndTypeClientTest.java @@ -40,10 +40,14 @@ public class WidgetBundleAndTypeClientTest extends AbstractContainerTest { @Test public void testWidgetsBundles_verifyInitialSetup() { + performTestOnEachEdge(this::_testWidgetsBundles_verifyInitialSetup); + } + + private void _testWidgetsBundles_verifyInitialSetup() { Awaitility.await() .pollInterval(500, TimeUnit.MILLISECONDS) .atMost(30, TimeUnit.SECONDS) - .until(() -> edgeRestClient.getWidgetsBundles(new PageLink(100)).getTotalElements() == 28); + .until(() -> edgeRestClient.getWidgetsBundles(new PageLink(100)).getTotalElements() == 30); PageData pageData = edgeRestClient.getWidgetsBundles(new PageLink(100)); assertEntitiesByIdsAndType(pageData.getData().stream().map(IdBased::getId).collect(Collectors.toList()), EntityType.WIDGETS_BUNDLE); @@ -68,6 +72,10 @@ public void testWidgetsBundles_verifyInitialSetup() { @Test public void testWidgetsBundleAndWidgetType() { + performTestOnEachEdge(this::_testWidgetsBundleAndWidgetType); + } + + private void _testWidgetsBundleAndWidgetType() { // create widget bundle WidgetsBundle widgetsBundle = new WidgetsBundle(); widgetsBundle.setTitle("Test Widget Bundle"); diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 2008d21b1a..00e7bc0e97 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -3918,7 +3918,7 @@ public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean return restTemplate.postForEntity(baseURL + "/api/otaPackage?isUrl={isUrl}", otaPackageInfo, OtaPackageInfo.class, params).getBody(); } - public OtaPackageInfo saveOtaPackageData(OtaPackageId otaPackageId, String checkSum, ChecksumAlgorithm checksumAlgorithm, String fileName, byte[] fileBytes) throws Exception { + public OtaPackageInfo saveOtaPackageData(OtaPackageId otaPackageId, String checkSum, ChecksumAlgorithm checksumAlgorithm, String fileName, byte[] fileBytes) { HttpEntity> requestEntity = createMultipartRequest(fileName, fileBytes, null, Collections.emptyMap()); Map params = new HashMap<>();