From 7bc43cb7bfa7134cc60a5d9051810ff45993b2ea Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 8 Dec 2023 12:33:43 +0200 Subject: [PATCH 01/11] Inline images when creating version --- .../service/sync/ie/DefaultEntitiesExportImportService.java | 4 ++-- .../sync/ie/exporting/impl/AssetProfileExportService.java | 1 + .../sync/ie/exporting/impl/DashboardExportService.java | 1 + .../sync/ie/exporting/impl/DefaultEntityExportService.java | 3 +++ .../sync/ie/exporting/impl/DeviceProfileExportService.java | 1 + .../sync/ie/exporting/impl/WidgetTypeExportService.java | 5 +++-- .../sync/ie/exporting/impl/WidgetsBundleExportService.java | 1 + 7 files changed, 12 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 1cf0f0fc4d..bc1735f30b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -65,8 +65,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final TbNotificationEntityService entityNotificationService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( - EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.DASHBOARD, - EntityType.ASSET_PROFILE, EntityType.ASSET, + EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.TB_RESOURCE, + EntityType.DASHBOARD, EntityType.ASSET_PROFILE, EntityType.ASSET, EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.ENTITY_VIEW, EntityType.WIDGET_TYPE, EntityType.WIDGETS_BUNDLE, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java index 710e33f106..d8a3e76b7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java @@ -34,6 +34,7 @@ protected void setRelatedEntities(EntitiesExportCtx ctx, AssetProfile assetPr assetProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultDashboardId())); assetProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultRuleChainId())); assetProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultEdgeRuleChainId())); + imageService.inlineImage(assetProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index d1278bc585..e2fd8e1666 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -48,6 +48,7 @@ protected void setRelatedEntities(EntitiesExportCtx ctx, Dashboard dashboard, for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.emptySet(), WIDGET_CONFIG_PROCESSED_FIELDS_PATTERN); } + imageService.inlineImages(dashboard); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index a9e0cf1166..0caea1f9a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; @@ -58,6 +59,8 @@ public class DefaultEntityExportService ctx, I entityId) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java index b95d8b83e2..8c6269831d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java @@ -34,6 +34,7 @@ protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile device deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultEdgeRuleChainId())); + imageService.inlineImage(deviceProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java index 5edc9782d4..5ff5e88700 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java @@ -32,10 +32,11 @@ public class WidgetTypeExportService extends BaseEntityExportService { @Override - protected void setRelatedEntities(EntitiesExportCtx ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData) { - if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { + protected void setRelatedEntities(EntitiesExportCtx ctx, WidgetTypeDetails widgetTypeDetails, WidgetTypeExportData exportData) { + if (widgetTypeDetails.getTenantId() == null || widgetTypeDetails.getTenantId().isNullUid()) { throw new IllegalArgumentException("Export of system Widget Type is not allowed"); } + imageService.inlineImages(widgetTypeDetails); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java index f984bb9d79..46078ac293 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -41,6 +41,7 @@ protected void setRelatedEntities(EntitiesExportCtx ctx, WidgetsBundle widget if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { throw new IllegalArgumentException("Export of system Widget Bundles is not allowed"); } + imageService.inlineImage(widgetsBundle); List fqns = widgetTypeService.findWidgetFqnsByWidgetsBundleId(ctx.getTenantId(), widgetsBundle.getId()); exportData.setFqns(fqns); From b9ac141bffae2c71c1287cb08bc90ed379c6e006 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 8 Dec 2023 12:34:24 +0200 Subject: [PATCH 02/11] When replacing base64 with links, keep the transaction with no rollback --- .../server/dao/resource/BaseImageService.java | 9 +++++---- .../dao/sql/resource/TbResourceInfoRepository.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index ad3934f76e..29f9ba8ae0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -27,7 +27,6 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Base64Utils; import org.thingsboard.common.util.JacksonUtil; @@ -273,7 +272,7 @@ public TbResourceInfo findSystemOrTenantImageByEtag(TenantId tenantId, String et return resourceInfoDao.findSystemOrTenantImageByEtag(tenantId, ResourceType.IMAGE, etag); } - @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure + @Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure @Override public boolean replaceBase64WithImageUrl(HasImage entity, String type) { log.trace("Executing replaceBase64WithImageUrl [{}] [{}] [{}]", entity.getTenantId(), type, entity.getName()); @@ -288,7 +287,7 @@ public boolean replaceBase64WithImageUrl(HasImage entity, String type) { return result.isUpdated(); } - @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure + @Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure @Override public boolean replaceBase64WithImageUrl(WidgetTypeDetails entity) { log.trace("Executing replaceBase64WithImageUrl [{}] [WidgetTypeDetails] [{}]", entity.getTenantId(), entity.getId()); @@ -314,7 +313,7 @@ public boolean replaceBase64WithImageUrl(WidgetTypeDetails entity) { return updated; } - @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure + @Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure @Override public boolean replaceBase64WithImageUrl(Dashboard entity) { log.trace("Executing replaceBase64WithImageUrl [{}] [Dashboard] [{}]", entity.getTenantId(), entity.getId()); @@ -462,6 +461,8 @@ private UpdateResult base64ToImageUrl(TenantId tenantId, String name, String dat } return UpdateResult.of(false, data); } + } else { + log.debug("[{}] Using existing image {} ({} - '{}') for '{}'", tenantId, imageInfo.getResourceKey(), imageInfo.getTenantId(), imageInfo.getName(), name); } return UpdateResult.of(true, DataConstants.TB_IMAGE_PREFIX + imageInfo.getLink()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java index ced94e1ea8..62d9ec5155 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java @@ -66,7 +66,7 @@ Set findKeysByTenantIdAndResourceTypeAndResourceKeyStartingWith(@Param(" List findByTenantIdAndEtagAndResourceKeyStartingWith(UUID tenantId, String etag, String query); @Query(value = "SELECT * FROM resource r WHERE (r.tenant_id = '13814000-1dd2-11b2-8080-808080808080' OR r.tenant_id = :tenantId) " + - "AND r.resource_type = :resourceType AND r.etag = :etag LIMIT 1", nativeQuery = true) + "AND r.resource_type = :resourceType AND r.etag = :etag ORDER BY created_time, id LIMIT 1", nativeQuery = true) TbResourceInfoEntity findSystemOrTenantImageByEtag(@Param("tenantId") UUID tenantId, @Param("resourceType") String resourceType, @Param("etag") String etag); From 2378f2ec2dbeb8ddf013d46a433d9c9671942c07 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 8 Dec 2023 13:46:42 +0200 Subject: [PATCH 03/11] Add handling logic for backward compatibility with previous Edges --- .../asset/AssetMsgConstructorV1.java | 6 +++ .../dashboard/DashboardMsgConstructorV1.java | 6 +++ .../device/DeviceMsgConstructorV1.java | 5 +++ .../resource/ResourceMsgConstructorV1.java | 7 +++- .../tenant/TenantMsgConstructorV1.java | 2 +- .../widget/WidgetMsgConstructorV1.java | 7 ++++ .../resource/ResourceEdgeProcessor.java | 4 +- .../resource/ResourceEdgeProcessorV1.java | 2 +- .../server/dao/resource/ImageService.java | 8 ++++ .../server/dao/resource/BaseImageService.java | 41 +++++++++++++++---- 10 files changed, 76 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java index 95e1e0e597..ddf975d4ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/asset/AssetMsgConstructorV1.java @@ -16,10 +16,12 @@ package org.thingsboard.server.service.edge.rpc.constructor.asset; import com.google.protobuf.ByteString; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; @@ -31,6 +33,9 @@ @TbCoreComponent public class AssetMsgConstructorV1 extends BaseAssetMsgConstructor { + @Autowired + private ImageService imageService; + @Override public AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asset) { AssetUpdateMsg.Builder builder = AssetUpdateMsg.newBuilder() @@ -58,6 +63,7 @@ public AssetUpdateMsg constructAssetUpdatedMsg(UpdateMsgType msgType, Asset asse @Override public AssetProfileUpdateMsg constructAssetProfileUpdatedMsg(UpdateMsgType msgType, AssetProfile assetProfile) { + imageService.inlineImageForEdge(assetProfile); AssetProfileUpdateMsg.Builder builder = AssetProfileUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(assetProfile.getId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java index 6a3bb43797..548b1e1fd0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/dashboard/DashboardMsgConstructorV1.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.edge.rpc.constructor.dashboard; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -26,8 +28,12 @@ @TbCoreComponent public class DashboardMsgConstructorV1 extends BaseDashboardMsgConstructor { + @Autowired + private ImageService imageService; + @Override public DashboardUpdateMsg constructDashboardUpdatedMsg(UpdateMsgType msgType, Dashboard dashboard) { + imageService.inlineImagesForEdge(dashboard); DashboardUpdateMsg.Builder builder = DashboardUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(dashboard.getId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java index 4442387b21..04f23f25b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/device/DeviceMsgConstructorV1.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; @@ -38,6 +39,9 @@ public class DeviceMsgConstructorV1 extends BaseDeviceMsgConstructor { @Autowired private DataDecodingEncodingService dataDecodingEncodingService; + @Autowired + private ImageService imageService; + @Override public DeviceUpdateMsg constructDeviceUpdatedMsg(UpdateMsgType msgType, Device device) { DeviceUpdateMsg.Builder builder = DeviceUpdateMsg.newBuilder() @@ -91,6 +95,7 @@ public DeviceCredentialsUpdateMsg constructDeviceCredentialsUpdatedMsg(DeviceCre @Override public DeviceProfileUpdateMsg constructDeviceProfileUpdatedMsg(UpdateMsgType msgType, DeviceProfile deviceProfile) { + imageService.inlineImageForEdge(deviceProfile); DeviceProfileUpdateMsg.Builder builder = DeviceProfileUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(deviceProfile.getId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java index d110076bea..f58f0b7c25 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/resource/ResourceMsgConstructorV1.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.edge.rpc.constructor.resource; -import com.google.protobuf.ByteString; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; @@ -29,6 +29,11 @@ public class ResourceMsgConstructorV1 extends BaseResourceMsgConstructor { @Override public ResourceUpdateMsg constructResourceUpdatedMsg(UpdateMsgType msgType, TbResource tbResource) { + if (ResourceType.IMAGE.equals(tbResource.getResourceType())) { + // Exclude support for a recently added resource type when dealing with older Edges + // to maintain compatibility and avoid potential issues. + return null; + } ResourceUpdateMsg.Builder builder = ResourceUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(tbResource.getId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java index a3f815cc18..b454a3b971 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/tenant/TenantMsgConstructorV1.java @@ -78,7 +78,7 @@ public TenantUpdateMsg constructTenantUpdateMsg(UpdateMsgType msgType, Tenant te @Override public TenantProfileUpdateMsg constructTenantProfileUpdateMsg(UpdateMsgType msgType, TenantProfile tenantProfile, EdgeVersion edgeVersion) { - ByteString profileData = EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_6_1) ? + ByteString profileData = EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_6_2) ? ByteString.empty() : ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile.getProfileData())); TenantProfileUpdateMsg.Builder builder = TenantProfileUpdateMsg.newBuilder() .setMsgType(msgType) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java index 67bdbf1f54..1bc6857863 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/widget/WidgetMsgConstructorV1.java @@ -16,11 +16,13 @@ package org.thingsboard.server.service.edge.rpc.constructor.widget; import com.google.protobuf.ByteString; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; @@ -36,8 +38,12 @@ @TbCoreComponent public class WidgetMsgConstructorV1 extends BaseWidgetMsgConstructor { + @Autowired + private ImageService imageService; + @Override public WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgType, WidgetsBundle widgetsBundle, List widgets) { + imageService.inlineImageForEdge(widgetsBundle); WidgetsBundleUpdateMsg.Builder builder = WidgetsBundleUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(widgetsBundle.getId().getId().getMostSignificantBits()) @@ -62,6 +68,7 @@ public WidgetsBundleUpdateMsg constructWidgetsBundleUpdateMsg(UpdateMsgType msgT @Override public WidgetTypeUpdateMsg constructWidgetTypeUpdateMsg(UpdateMsgType msgType, WidgetTypeDetails widgetTypeDetails, EdgeVersion edgeVersion) { + imageService.inlineImagesForEdge(widgetTypeDetails); WidgetTypeUpdateMsg.Builder builder = WidgetTypeUpdateMsg.newBuilder() .setMsgType(msgType) .setIdMSB(widgetTypeDetails.getId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java index f06991fd0f..f0da087472 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java @@ -81,10 +81,10 @@ public DownlinkMsg convertResourceEventToDownlink(EdgeEvent edgeEvent, EdgeVersi UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); ResourceUpdateMsg resourceUpdateMsg = ((ResourceMsgConstructor) resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructResourceUpdatedMsg(msgType, tbResource); - downlinkMsg = DownlinkMsg.newBuilder() + downlinkMsg = resourceUpdateMsg != null ? DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addResourceUpdateMsg(resourceUpdateMsg) - .build(); + .build() : null; } break; case DELETED: diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java index dd105729fe..56b1d5507c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessorV1.java @@ -41,7 +41,7 @@ protected TbResource constructResourceFromUpdateMsg(TenantId tenantId, TbResourc resource.setResourceKey(resourceUpdateMsg.getResourceKey()); resource.setResourceType(ResourceType.valueOf(resourceUpdateMsg.getResourceType())); resource.setFileName(resourceUpdateMsg.getFileName()); - resource.setData(resourceUpdateMsg.hasData() ? resourceUpdateMsg.getData().getBytes() : null); + resource.setEncodedData(resourceUpdateMsg.hasData() ? resourceUpdateMsg.getData() : null); resource.setEtag(resourceUpdateMsg.hasEtag() ? resourceUpdateMsg.getEtag() : null); return resource; } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java index f51f83d2f0..7e5542919b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java @@ -47,7 +47,9 @@ public interface ImageService { TbResourceInfo findSystemOrTenantImageByEtag(TenantId tenantId, String etag); boolean replaceBase64WithImageUrl(HasImage entity, String type); + boolean replaceBase64WithImageUrl(Dashboard dashboard); + boolean replaceBase64WithImageUrl(WidgetTypeDetails widgetType); void inlineImage(HasImage entity); @@ -55,4 +57,10 @@ public interface ImageService { void inlineImages(Dashboard dashboard); void inlineImages(WidgetTypeDetails widgetTypeDetails); + + void inlineImageForEdge(HasImage entity); + + void inlineImagesForEdge(Dashboard dashboard); + + void inlineImagesForEdge(WidgetTypeDetails widgetTypeDetails); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index ad3934f76e..b2da580009 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -517,17 +517,37 @@ public void inlineImage(HasImage entity) { public void inlineImages(Dashboard dashboard) { log.trace("Executing inlineImage [{}] [Dashboard] [{}]", dashboard.getTenantId(), dashboard.getId()); inlineImage(dashboard); - inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration()); + inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration(), true); } @Override public void inlineImages(WidgetTypeDetails widgetTypeDetails) { log.trace("Executing inlineImage [{}] [WidgetTypeDetails] [{}]", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId()); inlineImage(widgetTypeDetails); - inlineIntoJson(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor()); + inlineIntoJson(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), true); } - private void inlineIntoJson(TenantId tenantId, JsonNode root) { + @Override + public void inlineImageForEdge(HasImage entity) { + log.trace("Executing inlineImageForEdge [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName()); + entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), false)); + } + + @Override + public void inlineImagesForEdge(Dashboard dashboard) { + log.trace("Executing inlineImagesForEdge [{}] [Dashboard] [{}]", dashboard.getTenantId(), dashboard.getId()); + inlineImageForEdge(dashboard); + inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration(), false); + } + + @Override + public void inlineImagesForEdge(WidgetTypeDetails widgetTypeDetails) { + log.trace("Executing inlineImage [{}] [WidgetTypeDetails] [{}]", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId()); + inlineImageForEdge(widgetTypeDetails); + inlineIntoJson(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), false); + } + + private void inlineIntoJson(TenantId tenantId, JsonNode root, boolean addTbImagePrefix) { Queue tasks = new LinkedList<>(); tasks.add(new JsonNodeProcessingTask("", root)); while (!tasks.isEmpty()) { @@ -543,7 +563,7 @@ private void inlineIntoJson(TenantId tenantId, JsonNode root) { String childName = it.next(); JsonNode childValue = on.get(childName); if (childValue.isTextual()) { - on.put(childName, inlineImage(tenantId, currentPath + childName, childValue.asText())); + on.put(childName, inlineImage(tenantId, currentPath + childName, childValue.asText(), addTbImagePrefix)); } else if (childValue.isObject() || childValue.isArray()) { tasks.add(new JsonNodeProcessingTask(currentPath + childName, childValue)); } @@ -555,7 +575,7 @@ private void inlineIntoJson(TenantId tenantId, JsonNode root) { if (element.isObject()) { tasks.add(new JsonNodeProcessingTask(currentPath + "." + i, element)); } else if (element.isTextual()) { - childArray.set(i, inlineImage(tenantId, currentPath + "." + i, element.asText())); + childArray.set(i, inlineImage(tenantId, currentPath + "." + i, element.asText(), addTbImagePrefix)); } } } @@ -563,6 +583,10 @@ private void inlineIntoJson(TenantId tenantId, JsonNode root) { } private String inlineImage(TenantId tenantId, String path, String url) { + return inlineImage(tenantId, path, url, true); + } + + private String inlineImage(TenantId tenantId, String path, String url, boolean addTbImagePrefix) { try { ImageCacheKey key = getKeyFromUrl(tenantId, url); if (key != null) { @@ -570,8 +594,11 @@ private String inlineImage(TenantId tenantId, String path, String url) { if (imageInfo != null) { byte[] data = key.isPreview() ? getImagePreview(tenantId, imageInfo.getId()) : getImageData(tenantId, imageInfo.getId()); ImageDescriptor descriptor = getImageDescriptor(imageInfo, key.isPreview()); - String tbImagePrefix = "tb-image:" + Base64Utils.encodeToString(imageInfo.getResourceKey().getBytes(StandardCharsets.UTF_8)) + ":" - + Base64Utils.encodeToString(imageInfo.getName().getBytes(StandardCharsets.UTF_8)) + ";"; + String tbImagePrefix = ""; + if (addTbImagePrefix) { + tbImagePrefix = "tb-image:" + Base64Utils.encodeToString(imageInfo.getResourceKey().getBytes(StandardCharsets.UTF_8)) + ":" + + Base64Utils.encodeToString(imageInfo.getName().getBytes(StandardCharsets.UTF_8)) + ";"; + } return tbImagePrefix + "data:" + descriptor.getMediaType() + ";base64," + Base64Utils.encodeToString(data); } } From c6559d6bbac0b0e95d19c562e987cfbe9dce69b8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 8 Dec 2023 13:57:49 +0200 Subject: [PATCH 04/11] Always load system images; update images only after loading system widgets --- .../server/install/ThingsboardInstallService.java | 7 +++++-- .../thingsboard/server/service/install/InstallScripts.java | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 3fc68b9cc3..5ab3b7b28e 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -271,9 +271,8 @@ public void performInstall() { case "3.6.1": log.info("Upgrading ThingsBoard from version 3.6.1 to 3.6.2 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.1"); - installScripts.loadSystemImages(); if (!getEnv("SKIP_IMAGES_MIGRATION", false)) { - installScripts.updateImages(); + installScripts.setUpdateImages(true); } else { log.info("Skipping images migration. Run the upgrade with fromVersion as '3.6.2-images' to migrate"); } @@ -288,6 +287,10 @@ public void performInstall() { dataUpdateService.upgradeRuleNodes(); systemDataLoaderService.loadSystemWidgets(); installScripts.loadSystemLwm2mResources(); + installScripts.loadSystemImages(); + if (installScripts.isUpdateImages()) { + installScripts.updateImages(); + } } log.info("Upgrade finished successfully!"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 598c3c8965..9373ac6d34 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.install; import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -113,6 +115,8 @@ public class InstallScripts { @Autowired private ImagesUpdater imagesUpdater; + @Getter @Setter + private boolean updateImages = false; Path getTenantRuleChainsDir() { return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, RULE_CHAINS_DIR); From c48737be47364e84d7a30cde1078dd12e8f878cc Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 8 Dec 2023 14:06:05 +0200 Subject: [PATCH 05/11] Change order for EdgeSyncCursor: up resource to top, cuz all entities which 'hasImage' should have link for resource --- .../thingsboard/server/service/edge/rpc/EdgeSyncCursor.java | 4 ++-- .../thingsboard/server/dao/resource/BaseImageService.java | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index 98c93e54ec..7427b0a6a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -56,6 +56,8 @@ public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge, boolean fullSync) { if (fullSync) { fetchers.add(new TenantEdgeEventFetcher(ctx.getTenantService())); fetchers.add(new QueuesEdgeEventFetcher(ctx.getQueueService())); + fetchers.add(new SystemResourcesEdgeEventFetcher(ctx.getResourceService())); + fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService())); fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService())); fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService())); fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService())); @@ -79,8 +81,6 @@ public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge, boolean fullSync) { fetchers.add(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new OtaPackagesEdgeEventFetcher(ctx.getOtaPackageService())); - fetchers.add(new SystemResourcesEdgeEventFetcher(ctx.getResourceService())); - fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService())); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index b2da580009..34c6c8c8f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -510,7 +510,7 @@ private boolean base64ToImageUrlRecursively(TenantId tenantId, String title, Jso @Override public void inlineImage(HasImage entity) { log.trace("Executing inlineImage [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName()); - entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage())); + entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), true)); } @Override @@ -582,10 +582,6 @@ private void inlineIntoJson(TenantId tenantId, JsonNode root, boolean addTbImage } } - private String inlineImage(TenantId tenantId, String path, String url) { - return inlineImage(tenantId, path, url, true); - } - private String inlineImage(TenantId tenantId, String path, String url, boolean addTbImagePrefix) { try { ImageCacheKey key = getKeyFromUrl(tenantId, url); From 0ae62e7a579a164a8b459009ae55a440f64495b9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 11 Dec 2023 17:42:04 +0200 Subject: [PATCH 06/11] Remove @Length validation for image fields --- .../java/org/thingsboard/server/common/data/DashboardInfo.java | 1 - .../java/org/thingsboard/server/common/data/DeviceProfile.java | 1 - .../org/thingsboard/server/common/data/asset/AssetProfile.java | 1 - .../server/common/data/widget/WidgetTypeDetails.java | 1 - .../thingsboard/server/common/data/widget/WidgetsBundle.java | 2 -- 5 files changed, 6 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java index 19b69f4f27..c3a06a0e6a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java @@ -38,7 +38,6 @@ public class DashboardInfo extends BaseData implements HasName, Has @NoXss @Length(fieldName = "title") private String title; - @Length(fieldName = "image", max = 1000000) private String image; @Valid private Set assignedCustomers; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index abee7284c8..be4426a1d4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -54,7 +54,6 @@ public class DeviceProfile extends BaseData implements HasName, @NoXss @ApiModelProperty(position = 11, value = "Device Profile description. ") private String description; - @Length(fieldName = "image", max = 1000000) @ApiModelProperty(position = 12, value = "Either URL or Base64 data of the icon. Used in the mobile application to visualize set of device profiles in the grid view. ") private String image; private boolean isDefault; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java index 96b24ad2f2..2daa64061d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java @@ -52,7 +52,6 @@ public class AssetProfile extends BaseData implements HasName, H @NoXss @ApiModelProperty(position = 11, value = "Asset Profile description. ") private String description; - @Length(fieldName = "image", max = 1000000) @ApiModelProperty(position = 12, value = "Either URL or Base64 data of the icon. Used in the mobile application to visualize set of asset profiles in the grid view. ") private String image; private boolean isDefault; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeDetails.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeDetails.java index b335038b54..02992421eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeDetails.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeDetails.java @@ -32,7 +32,6 @@ @JsonPropertyOrder({ "fqn", "name", "deprecated", "image", "description", "descriptor", "externalId" }) public class WidgetTypeDetails extends WidgetType implements HasName, HasTenantId, HasImage, ExportableEntity { - @Length(fieldName = "image", max = 1000000) @ApiModelProperty(position = 9, value = "Relative or external image URL. Replaced with image data URL (Base64) in case of relative URL and 'inlineImages' option enabled.") private String image; @NoXss diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java index ce0367f57d..5d38c10550 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java @@ -57,7 +57,6 @@ public class WidgetsBundle extends BaseData implements HasName, @ApiModelProperty(position = 5, value = "Title used in search and UI", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private String title; - @Length(fieldName = "image", max = 1000000) @Getter @Setter @ApiModelProperty(position = 6, value = "Relative or external image URL. Replaced with image data URL (Base64) in case of relative URL and 'inlineImages' option enabled.", accessMode = ApiModelProperty.AccessMode.READ_ONLY) @@ -66,7 +65,6 @@ public class WidgetsBundle extends BaseData implements HasName, @NoXss @Length(fieldName = "description", max = 1024) @Getter - @Setter @ApiModelProperty(position = 7, value = "Description", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private String description; From 27743c3e3ae524bc84e8ffab25ce0148f54d5039 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 11 Dec 2023 17:51:06 +0200 Subject: [PATCH 07/11] Evict image from cache when deleted --- .../server/service/resource/DefaultTbImageService.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java index cd69f884ac..1fbd62f88f 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java @@ -145,6 +145,15 @@ public TbImageDeleteResult delete(TbResourceInfo imageInfo, User user, boolean f TbImageDeleteResult result = imageService.deleteImage(imageInfo, force); if (result.isSuccess()) { notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.DELETED, user, imageId.toString()); + evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), false)); + evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), true)); + clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder() + .setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setResourceKey(imageInfo.getResourceKey()) + .build()) + .build()); } return result; } catch (Exception e) { From 2cfa040cc5213c5734c5e7c2e04a7619c99afb1e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 12 Dec 2023 11:55:57 +0200 Subject: [PATCH 08/11] Validate image size as soon as possible; pretty max size in the error --- .../server/controller/ImageController.java | 8 ++++- .../server/dao/service/DataValidator.java | 3 +- .../validator/ResourceDataValidator.java | 33 +++++++++++-------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/ImageController.java b/application/src/main/java/org/thingsboard/server/controller/ImageController.java index 7685640929..815adaca5f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ImageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ImageController.java @@ -50,9 +50,10 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.resource.ImageCacheKey; import org.thingsboard.server.dao.resource.ImageService; +import org.thingsboard.server.dao.service.validator.ResourceDataValidator; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.dao.resource.ImageCacheKey; import org.thingsboard.server.service.resource.TbImageService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -77,6 +78,8 @@ public class ImageController extends BaseController { private final ImageService imageService; private final TbImageService tbImageService; + private final ResourceDataValidator resourceValidator; + @Value("${cache.image.systemImagesBrowserTtlInMinutes:0}") private int systemImagesBrowserTtlInMinutes; @Value("${cache.image.tenantImagesBrowserTtlInMinutes:0}") @@ -94,6 +97,7 @@ public TbResourceInfo uploadImage(@RequestPart MultipartFile file, TbResource image = new TbResource(); image.setTenantId(user.getTenantId()); accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.CREATE, null, image); + resourceValidator.validateResourceSize(user.getTenantId(), null, file.getSize()); image.setFileName(file.getOriginalFilename()); if (StringUtils.isNotEmpty(title)) { @@ -115,6 +119,8 @@ public TbResourceInfo updateImage(@PathVariable String type, @PathVariable String key, @RequestPart MultipartFile file) throws Exception { TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE); + resourceValidator.validateResourceSize(getTenantId(), imageInfo.getId(), file.getSize()); + TbResource image = new TbResource(imageInfo); image.setData(file.getBytes()); image.setFileName(file.getOriginalFilename()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index 961fffcdc0..80c9376f1c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.BaseData; @@ -129,7 +130,7 @@ protected void validateMaxSumDataSizePerTenant(TenantId tenantId, EntityType entityType) { if (maxSumDataSize > 0) { if (dataDao.sumDataSizeByTenantId(tenantId) + currentDataSize > maxSumDataSize) { - throw new DataValidationException(String.format("%ss total size exceeds the maximum of " + maxSumDataSize + " bytes", entityType.getNormalName())); + throw new DataValidationException(String.format("%ss total size exceeds the maximum of " + FileUtils.byteCountToDisplaySize(maxSumDataSize), entityType.getNormalName())); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java index 26d8ab75fb..352966d561 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.service.validator; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.widget.BaseWidgetType; @@ -83,19 +85,8 @@ protected void validateDataImpl(TenantId tenantId, TbResource resource) { if (resource.getResourceType() == null) { throw new DataValidationException("Resource type should be specified!"); } - if (!resource.getTenantId().isSysTenantId() && resource.getData() != null) { - DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); - long maxResourceSize = profileConfiguration.getMaxResourceSize(); - if (maxResourceSize > 0 && resource.getData().length > maxResourceSize) { - throw new IllegalArgumentException("Resource exceeds the maximum size of " + maxResourceSize + " bytes"); - } - long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes(); - int dataSize = resource.getData().length; - if (resource.getId() != null) { - long prevSize = resourceDao.getResourceSize(tenantId, resource.getId()); - dataSize -= prevSize; - } - validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE); + if (resource.getData() != null) { + validateResourceSize(resource.getTenantId(), resource.getId(), resource.getData().length); } if (StringUtils.isEmpty(resource.getFileName())) { throw new DataValidationException("Resource file name should be specified!"); @@ -108,6 +99,22 @@ protected void validateDataImpl(TenantId tenantId, TbResource resource) { } } + public void validateResourceSize(TenantId tenantId, TbResourceId resourceId, long dataSize) { + if (!tenantId.isSysTenantId()) { + DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); + long maxResourceSize = profileConfiguration.getMaxResourceSize(); + if (maxResourceSize > 0 && dataSize > maxResourceSize) { + throw new IllegalArgumentException("Resource exceeds the maximum size of " + FileUtils.byteCountToDisplaySize(maxResourceSize)); + } + long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes(); + if (resourceId != null) { + long prevSize = resourceDao.getResourceSize(tenantId, resourceId); + dataSize -= prevSize; + } + validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE); + } + } + @Override public void validateDelete(TenantId tenantId, EntityId resourceId) { List widgets = widgetTypeDao.findWidgetTypesInfosByTenantIdAndResourceId(tenantId.getId(), From c07e7063a99dd1089648f76dfcdf785d24bc5865 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 12 Dec 2023 18:24:09 +0200 Subject: [PATCH 09/11] Do not push system resources from cloud to edge --- .../service/edge/rpc/EdgeSyncCursor.java | 4 +- .../fetch/BaseResourceEdgeEventFetcher.java | 49 ------------------- .../SystemResourcesEdgeEventFetcher.java | 34 ------------- .../TenantResourcesEdgeEventFetcher.java | 21 ++++++-- 4 files changed, 17 insertions(+), 91 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseResourceEdgeEventFetcher.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/SystemResourcesEdgeEventFetcher.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index 7427b0a6a8..0fc5c93388 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.edge.rpc.fetch.OtaPackagesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.QueuesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.RuleChainsEdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.fetch.SystemResourcesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.SystemWidgetTypesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.SystemWidgetsBundlesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.TenantAdminUsersEdgeEventFetcher; @@ -56,8 +55,6 @@ public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge, boolean fullSync) { if (fullSync) { fetchers.add(new TenantEdgeEventFetcher(ctx.getTenantService())); fetchers.add(new QueuesEdgeEventFetcher(ctx.getQueueService())); - fetchers.add(new SystemResourcesEdgeEventFetcher(ctx.getResourceService())); - fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService())); fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService())); fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService())); fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService())); @@ -81,6 +78,7 @@ public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge, boolean fullSync) { fetchers.add(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new OtaPackagesEdgeEventFetcher(ctx.getOtaPackageService())); + fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseResourceEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseResourceEdgeEventFetcher.java deleted file mode 100644 index 5e0ea661e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/BaseResourceEdgeEventFetcher.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.fetch; - -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.EdgeUtils; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.edge.EdgeEventType; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.resource.ResourceService; - -@Slf4j -@AllArgsConstructor -public abstract class BaseResourceEdgeEventFetcher extends BasePageableEdgeEventFetcher { - - protected final ResourceService resourceService; - - @Override - PageData fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) { - return findTenantResources(tenantId, pageLink); - } - - @Override - EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, TbResource tbResource) { - return EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.TB_RESOURCE, - EdgeEventActionType.ADDED, tbResource.getId(), null); - } - - protected abstract PageData findTenantResources(TenantId tenantId, PageLink pageLink); -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/SystemResourcesEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/SystemResourcesEdgeEventFetcher.java deleted file mode 100644 index 42700ded3e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/SystemResourcesEdgeEventFetcher.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.fetch; - -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.resource.ResourceService; - -public class SystemResourcesEdgeEventFetcher extends BaseResourceEdgeEventFetcher { - - public SystemResourcesEdgeEventFetcher(ResourceService resourceService) { - super(resourceService); - } - - @Override - protected PageData findTenantResources(TenantId tenantId, PageLink pageLink) { - return resourceService.findAllTenantResources(TenantId.SYS_TENANT_ID, pageLink); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantResourcesEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantResourcesEdgeEventFetcher.java index 0992565285..61a6ce4e6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantResourcesEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantResourcesEdgeEventFetcher.java @@ -15,22 +15,33 @@ */ package org.thingsboard.server.service.edge.rpc.fetch; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.resource.ResourceService; +@AllArgsConstructor @Slf4j -public class TenantResourcesEdgeEventFetcher extends BaseResourceEdgeEventFetcher { +public class TenantResourcesEdgeEventFetcher extends BasePageableEdgeEventFetcher { - public TenantResourcesEdgeEventFetcher(ResourceService resourceService) { - super(resourceService); - } + private final ResourceService resourceService; @Override - protected PageData findTenantResources(TenantId tenantId, PageLink pageLink) { + PageData fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) { return resourceService.findAllTenantResources(tenantId, pageLink); } + + @Override + EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, TbResource tbResource) { + return EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.TB_RESOURCE, + EdgeEventActionType.ADDED, tbResource.getId(), null); + } } From 153d34cfd2d49a10e11c225a5d7d8701200eace7 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 12 Dec 2023 18:24:55 +0200 Subject: [PATCH 10/11] Create system resources on install/update and replace with cloud id during runtime --- .../install/ThingsboardInstallService.java | 9 ++--- .../rpc/processor/ResourceCloudProcessor.java | 8 ---- .../processor/WidgetTypeCloudProcessor.java | 39 ++----------------- .../install/update/DataUpdateService.java | 2 - .../update/DefaultDataUpdateService.java | 29 -------------- 5 files changed, 7 insertions(+), 80 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index b7c66a0319..786998deab 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -295,13 +295,10 @@ public void performInstall() { entityDatabaseSchemaService.createOrUpdateViewsAndFunctions(); entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry); - // edge specific - dataUpdateService.deleteAllWidgetBundlesAndTypes(); - // @voba - system widgets update is not required - uploaded from cloud // log.info("Updating system data..."); // dataUpdateService.upgradeRuleNodes(); - // systemDataLoaderService.loadSystemWidgets(); + systemDataLoaderService.loadSystemWidgets(); // installScripts.loadSystemLwm2mResources(); } @@ -338,7 +335,7 @@ public void performInstall() { systemDataLoaderService.createDefaultTenantProfiles(); systemDataLoaderService.createAdminSettings(); systemDataLoaderService.createRandomJwtSettings(); - // systemDataLoaderService.loadSystemWidgets(); + systemDataLoaderService.loadSystemWidgets(); // systemDataLoaderService.createOAuth2Templates(); // systemDataLoaderService.createQueues(); // systemDataLoaderService.createDefaultNotificationConfigs(); @@ -346,7 +343,7 @@ public void performInstall() { // systemDataLoaderService.loadSystemPlugins(); // systemDataLoaderService.loadSystemRules(); // installScripts.loadSystemLwm2mResources(); - // installScripts.loadSystemImages(); + installScripts.loadSystemImages(); if (loadDemo) { // log.info("Loading demo data..."); diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/ResourceCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/ResourceCloudProcessor.java index 821cf7309b..5901e8591a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/ResourceCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/ResourceCloudProcessor.java @@ -118,14 +118,6 @@ public UplinkMsg convertResourceEventToUplink(CloudEvent cloudEvent, EdgeVersion .build(); } break; - case DELETED: - ResourceUpdateMsg resourceUpdateMsg = ((ResourceMsgConstructor) - resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructResourceDeleteMsg(tbResourceId); - msg = UplinkMsg.newBuilder() - .setUplinkMsgId(EdgeUtils.nextPositiveInt()) - .addResourceUpdateMsg(resourceUpdateMsg) - .build(); - break; } return msg; } diff --git a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/WidgetTypeCloudProcessor.java b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/WidgetTypeCloudProcessor.java index 0807b32f6a..960eee61c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/WidgetTypeCloudProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/cloud/rpc/processor/WidgetTypeCloudProcessor.java @@ -22,18 +22,11 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.widget.DeprecatedFilter; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.common.data.widget.WidgetTypeInfo; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; -import java.util.List; import java.util.UUID; @Component @@ -53,17 +46,11 @@ public ListenableFuture processWidgetTypeMsgFromCloud(TenantId tenantId, W if (widgetTypeDetails == null) { throw new RuntimeException("[{" + tenantId + "}] widgetTypeUpdateMsg {" + widgetTypeUpdateMsg + "} cannot be converted to widget type"); } - try { - widgetTypeService.saveWidgetType(widgetTypeDetails, false); - } catch (Exception e) { - if (e instanceof DataValidationException && e.getMessage().contains("fqn already exists")) { - deleteWidgetBundlesAndTypes(TenantId.SYS_TENANT_ID); - deleteWidgetBundlesAndTypes(tenantId); - widgetTypeService.saveWidgetType(widgetTypeDetails, false); - } else { - throw new RuntimeException(e); - } + WidgetType widgetTypeByTenantIdAndFqn = widgetTypeService.findWidgetTypeByTenantIdAndFqn(widgetTypeDetails.getTenantId(), widgetTypeDetails.getFqn()); + if (widgetTypeByTenantIdAndFqn != null && !widgetTypeByTenantIdAndFqn.getId().equals(widgetTypeDetails.getId())) { + widgetTypeService.deleteWidgetType(widgetTypeByTenantIdAndFqn.getTenantId(), widgetTypeByTenantIdAndFqn.getId()); } + widgetTypeService.saveWidgetType(widgetTypeDetails, false); } finally { widgetCreationLock.unlock(); } @@ -82,22 +69,4 @@ public ListenableFuture processWidgetTypeMsgFromCloud(TenantId tenantId, W } return Futures.immediateFuture(null); } - - private void deleteWidgetBundlesAndTypes(TenantId tenantId) { - List systemWidgetsBundles = widgetsBundleService.findSystemWidgetsBundles(tenantId); - for (WidgetsBundle systemWidgetsBundle : systemWidgetsBundles) { - if (systemWidgetsBundle != null) { - PageData widgetTypes; - var pageLink = new PageLink(1024); - do { - widgetTypes = widgetTypeService.findWidgetTypesInfosByWidgetsBundleId(tenantId, systemWidgetsBundle.getId(), false, DeprecatedFilter.ALL, null, pageLink); - for (var widgetType : widgetTypes.getData()) { - widgetTypeService.deleteWidgetType(tenantId, widgetType.getId()); - } - pageLink.nextPageLink(); - } while (widgetTypes.hasNext()); - widgetsBundleService.deleteWidgetsBundle(tenantId, systemWidgetsBundle.getId()); - } - } - } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DataUpdateService.java index 3da04b3117..289e3b1c94 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DataUpdateService.java @@ -20,6 +20,4 @@ public interface DataUpdateService { void updateData(String fromVersion) throws Exception; void upgradeRuleNodes(); - - void deleteAllWidgetBundlesAndTypes(); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 394e88fd1b..6135bb83da 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -68,8 +68,6 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.common.data.widget.DeprecatedFilter; -import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; @@ -271,33 +269,6 @@ public void updateData(String fromVersion) throws Exception { } } - @Override - public void deleteAllWidgetBundlesAndTypes() { - PageData tenants = tenantService.findTenants(new PageLink(Integer.MAX_VALUE)); - for (Tenant tenant : tenants.getData()) { - deleteWidgetBundlesAndTypes(tenant.getId()); - } - deleteWidgetBundlesAndTypes(TenantId.SYS_TENANT_ID); - } - - private void deleteWidgetBundlesAndTypes(TenantId tenantId) { - List systemWidgetsBundles = widgetsBundleService.findSystemWidgetsBundles(tenantId); - for (WidgetsBundle systemWidgetsBundle : systemWidgetsBundles) { - if (systemWidgetsBundle != null) { - PageData widgetTypes; - var pageLink = new PageLink(1024); - do { - widgetTypes = widgetTypeService.findWidgetTypesInfosByWidgetsBundleId(tenantId, systemWidgetsBundle.getId(), false, DeprecatedFilter.ALL, null, pageLink); - for (var widgetType : widgetTypes.getData()) { - widgetTypeService.deleteWidgetType(tenantId, widgetType.getId()); - } - pageLink.nextPageLink(); - } while (widgetTypes.hasNext()); - widgetsBundleService.deleteWidgetsBundle(tenantId, systemWidgetsBundle.getId()); - } - } - } - private void migrateEdgeEvents(String logPrefix) { boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); if (!skipEdgeEventsMigration) { From 14ee8c4c8cbc7af25cba3ac57a7dddc4f20428bc Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 12 Dec 2023 18:44:08 +0200 Subject: [PATCH 11/11] Load system widgets on install/upgrade --- .../install/ThingsboardInstallService.java | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 7ad1a366f5..6536168e18 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -118,86 +118,86 @@ public void performInstall() { switch (upgradeFromVersion) { /* merge comment case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion - log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 1.2.3 to 1.3.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("1.2.3"); case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion - log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 1.3.0 to 1.3.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("1.3.0"); case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion - log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 1.3.1 to 1.4.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("1.3.1"); case "1.4.0": - log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 1.4.0 to 2.0.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("1.4.0"); dataUpdateService.updateData("1.4.0"); case "2.0.0": - log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.0.0 to 2.1.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.0.0"); case "2.1.1": - log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.1.1 to 2.1.2 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.1.1"); case "2.1.3": - log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.1.3 to 2.2.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.1.3"); case "2.3.0": - log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.3.0 to 2.3.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.3.0"); case "2.3.1": - log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.3.1 to 2.4.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.3.1"); case "2.4.0": - log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.4.0 to 2.4.1 ..."); case "2.4.1": - log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.4.1 to 2.4.2 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.4.1"); case "2.4.2": - log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.4.2 to 2.4.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("2.4.2"); case "2.4.3": - log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.4.3 to 2.5 ..."); if (databaseTsUpgradeService != null) { databaseTsUpgradeService.upgradeDatabase("2.4.3"); } databaseEntitiesUpgradeService.upgradeDatabase("2.4.3"); case "2.5.0": - log.info("Upgrading ThingsBoard from version 2.5.0 to 2.5.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.5.0 to 2.5.1 ..."); if (databaseTsUpgradeService != null) { databaseTsUpgradeService.upgradeDatabase("2.5.0"); } case "2.5.1": - log.info("Upgrading ThingsBoard from version 2.5.1 to 3.0.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 2.5.1 to 3.0.0 ..."); case "3.0.1": - log.info("Upgrading ThingsBoard from version 3.0.1 to 3.1.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.0.1 to 3.1.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.0.1"); dataUpdateService.updateData("3.0.1"); case "3.1.0": - log.info("Upgrading ThingsBoard from version 3.1.0 to 3.1.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.1.0 to 3.1.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.1.0"); case "3.1.1": - log.info("Upgrading ThingsBoard from version 3.1.1 to 3.2.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.1.1 to 3.2.0 ..."); if (databaseTsUpgradeService != null) { databaseTsUpgradeService.upgradeDatabase("3.1.1"); } @@ -205,16 +205,16 @@ public void performInstall() { dataUpdateService.updateData("3.1.1"); systemDataLoaderService.createOAuth2Templates(); case "3.2.0": - log.info("Upgrading ThingsBoard from version 3.2.0 to 3.2.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.2.0 to 3.2.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.2.0"); case "3.2.1": - log.info("Upgrading ThingsBoard from version 3.2.1 to 3.2.2 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.2.1 to 3.2.2 ..."); if (databaseTsUpgradeService != null) { databaseTsUpgradeService.upgradeDatabase("3.2.1"); } databaseEntitiesUpgradeService.upgradeDatabase("3.2.1"); case "3.2.2": - log.info("Upgrading ThingsBoard from version 3.2.2 to 3.3.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.2.2 to 3.3.0 ..."); if (databaseTsUpgradeService != null) { databaseTsUpgradeService.upgradeDatabase("3.2.2"); } @@ -223,11 +223,11 @@ public void performInstall() { dataUpdateService.updateData("3.2.2"); systemDataLoaderService.createOAuth2Templates(); case "3.3.0": - log.info("Upgrading ThingsBoard from version 3.3.0 to 3.3.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.3.0 to 3.3.1 ..."); case "3.3.1": - log.info("Upgrading ThingsBoard from version 3.3.1 to 3.3.2 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.3.1 to 3.3.2 ..."); case "3.3.2": - log.info("Upgrading ThingsBoard from version 3.3.2 to 3.3.3 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.3.2 to 3.3.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.3.2"); dataUpdateService.updateData("3.3.2"); */ @@ -240,17 +240,17 @@ public void performInstall() { databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); dataUpdateService.updateData("3.3.4"); case "3.4.0": - log.info("Upgrading ThingsBoard from version 3.4.0 to 3.4.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.4.0 to 3.4.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.4.0"); case "3.4.1": case "3.4.2": - log.info("Upgrading ThingsBoard from version 3.4.1 to 3.4.3 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.4.1 to 3.4.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.4.1"); dataUpdateService.updateData("3.4.1"); case "3.4.3": - log.info("Upgrading ThingsBoard from version 3.4.3 to 3.4.4 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.4.3 to 3.4.4 ..."); case "3.4.4": - log.info("Upgrading ThingsBoard from version 3.4.4 to 3.5.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.4.4 to 3.5.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.4.4"); dataUpdateService.updateData("3.4.4"); @@ -260,19 +260,19 @@ public void performInstall() { log.info("Skipping default notification configs creation"); } case "3.5.0": - log.info("Upgrading ThingsBoard from version 3.5.0 to 3.5.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.5.0 to 3.5.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.5.0"); case "3.5.1": - log.info("Upgrading ThingsBoard from version 3.5.1 to 3.6.0 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.5.1 to 3.6.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.5.1"); dataUpdateService.updateData("3.5.1"); systemDataLoaderService.updateDefaultNotificationConfigs(); case "3.6.0": - log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); dataUpdateService.updateData("3.6.0"); case "3.6.1": - log.info("Upgrading ThingsBoard from version 3.6.1 to 3.6.2 ..."); + log.info("Upgrading ThingsBoard Edge from version 3.6.1 to 3.6.2 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.1"); if (!getEnv("SKIP_IMAGES_MIGRATION", false)) { installScripts.setUpdateImages(true); @@ -294,8 +294,7 @@ public void performInstall() { entityDatabaseSchemaService.createOrUpdateViewsAndFunctions(); entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry); - // @voba - system widgets update is not required - uploaded from cloud - // log.info("Updating system data..."); + log.info("Updating system data..."); // dataUpdateService.upgradeRuleNodes(); systemDataLoaderService.loadSystemWidgets(); // installScripts.loadSystemLwm2mResources();