From 05561813328a70c340dada0adac68df56447c805 Mon Sep 17 00:00:00 2001 From: shiyiyue1102 Date: Wed, 18 Dec 2024 14:57:06 +0800 Subject: [PATCH] optimize history comparation (#12967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix type search on mysql model * 灰度模型迁移程序并发&迁移不落历史表 * Config migrate executor times * 1.Config migrate executor times 2. history comparation optimize * 1.Config migrate executor times 2. history comparation optimize * checkstyle --- .../server/controller/HistoryController.java | 20 -- .../controller/v2/HistoryControllerV2.java | 22 ++ ...Pair.java => ConfigHistoryInfoDetail.java} | 278 +++++++++--------- .../ConfigGrayModelMigrateService.java | 11 +- .../config/server/service/HistoryService.java | 130 +++++--- .../controller/HistoryControllerTest.java | 54 +--- .../v2/HistoryControllerV2Test.java | 27 +- .../server/service/HistoryServiceTest.java | 12 +- .../src/main/resources/application.properties | 10 +- .../mapper/HistoryConfigInfoMapper.java | 2 +- 10 files changed, 286 insertions(+), 280 deletions(-) rename config/src/main/java/com/alibaba/nacos/config/server/model/{ConfigHistoryInfoPair.java => ConfigHistoryInfoDetail.java} (58%) diff --git a/config/src/main/java/com/alibaba/nacos/config/server/controller/HistoryController.java b/config/src/main/java/com/alibaba/nacos/config/server/controller/HistoryController.java index 3de11b85ca..0aa7b2fae0 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/controller/HistoryController.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/controller/HistoryController.java @@ -21,7 +21,6 @@ import com.alibaba.nacos.common.utils.StringUtils; import com.alibaba.nacos.config.server.constant.Constants; import com.alibaba.nacos.config.server.model.ConfigHistoryInfo; -import com.alibaba.nacos.config.server.model.ConfigHistoryInfoPair; import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; import com.alibaba.nacos.config.server.paramcheck.ConfigDefaultHttpParamExtractor; import com.alibaba.nacos.core.paramcheck.ExtractorManager; @@ -102,25 +101,6 @@ public ConfigHistoryInfo getConfigHistoryInfo(@RequestParam("dataId") String dat return historyService.getConfigHistoryInfo(dataId, group, tenant, nid); } - /** - * Query the detailed configuration history information pair, including the original version and the updated version. notes: - * - * @param nid history_config_info nid - * @param dataId dataId @since 2.0.3 - * @param group groupId @since 2.0.3 - * @param tenant tenantId @since 2.0.3 - * @return history config info - * @since 2.0.3 add {@link Secured}, dataId, groupId and tenant for history config permission check. - */ - @GetMapping(value = "/pair") - @Secured(action = ActionTypes.READ, signType = SignType.CONFIG) - public ConfigHistoryInfoPair getConfigHistoryInfoPair(@RequestParam("dataId") String dataId, - @RequestParam("group") String group, - @RequestParam(value = "tenant", required = false, defaultValue = StringUtils.EMPTY) String tenant, - @RequestParam(value = "nid") Long nid) throws AccessException { - return historyService.getConfigHistoryInfoPair(dataId, group, tenant, nid); - } - /** * Query previous config history information. notes: * diff --git a/config/src/main/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2.java b/config/src/main/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2.java index 9cf181de19..978df0cf68 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2.java @@ -25,6 +25,7 @@ import com.alibaba.nacos.common.utils.StringUtils; import com.alibaba.nacos.config.server.constant.Constants; import com.alibaba.nacos.config.server.model.ConfigHistoryInfo; +import com.alibaba.nacos.config.server.model.ConfigHistoryInfoDetail; import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; import com.alibaba.nacos.config.server.paramcheck.ConfigDefaultHttpParamExtractor; import com.alibaba.nacos.core.paramcheck.ExtractorManager; @@ -112,6 +113,27 @@ public Result getConfigHistoryInfo(@RequestParam("dataId") St return Result.success(configHistoryInfo); } + + /** + * Query the detailed configuration history information pair, including the original version and the updated version. notes: + * + * @param nid history_config_info nid + * @param dataId dataId @since 2.0.3 + * @param group groupId @since 2.0.3 + * @param namespaceId namespaceId @since 2.0.3 + * @return history config info + * @since 2.0.3 add {@link Secured}, dataId, groupId and tenant for history config permission check. + */ + @GetMapping(value = "/detail") + @Secured(action = ActionTypes.READ, signType = SignType.CONFIG) + public Result getConfigHistoryInfoDetail(@RequestParam("dataId") String dataId, + @RequestParam("group") String group, + @RequestParam(value = "namespaceId", required = false, defaultValue = StringUtils.EMPTY) String namespaceId, + @RequestParam(value = "nid") Long nid) throws AccessException { + return Result.success(historyService.getConfigHistoryInfoDetail(dataId, group, namespaceId, nid)); + } + + /** * Query previous config history information. notes: * diff --git a/config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoPair.java b/config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoDetail.java similarity index 58% rename from config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoPair.java rename to config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoDetail.java index e3c64e2c07..807aa82a09 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoPair.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/model/ConfigHistoryInfoDetail.java @@ -16,238 +16,228 @@ package com.alibaba.nacos.config.server.model; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; - import java.io.Serializable; import java.sql.Timestamp; -import java.util.Objects; /** * ConfigHistoryInfoPair. * * @author dirtybit */ -public class ConfigHistoryInfoPair implements Serializable { - +public class ConfigHistoryInfoDetail implements Serializable { + private static final long serialVersionUID = -7827521105376245603L; - - @JsonSerialize(using = ToStringSerializer.class) + private long id; - + private long lastId = -1; - + private String dataId; - + private String group; - + private String tenant; - - private String appName; - - private String md5; - - private String content; - - private String updatedMd5; - - private String updatedContent; - - private String srcIp; - - private String srcUser; - + /** * Operation type, include inserting, updating and deleting. */ private String opType; - + private String publishType; - + private String grayName; - - private String extInfo; - + + private String appName; + + private String srcIp; + + private String srcUser; + + private String originalMd5; + + private String originalContent; + + private String originalEncryptedDataKey; + + private String originalExtInfo; + + private String updatedMd5; + + private String updatedContent; + + private String updatedEncryptedDataKey; + + private String updateExtInfo; + private Timestamp createdTime; - + private Timestamp lastModifiedTime; - - private String encryptedDataKey; - + public long getId() { return id; } - + public void setId(long id) { this.id = id; } - + public long getLastId() { return lastId; } - + public void setLastId(long lastId) { this.lastId = lastId; } - + public String getDataId() { return dataId; } - + public void setDataId(String dataId) { this.dataId = dataId; } - + public String getGroup() { return group; } - + public void setGroup(String group) { this.group = group; } - + public String getTenant() { return tenant; } - + public void setTenant(String tenant) { this.tenant = tenant; } - - public String getContent() { - return content; + + public String getOpType() { + return opType; } - - public void setContent(String content) { - this.content = content; + + public void setOpType(String opType) { + this.opType = opType; } - - public String getUpdatedMd5() { - return updatedMd5; + + public String getPublishType() { + return publishType; } - - public void setUpdatedMd5(String updatedMd5) { - this.updatedMd5 = updatedMd5; + + public void setPublishType(String publishType) { + this.publishType = publishType; } - - public String getUpdatedContent() { - return updatedContent; + + public String getGrayName() { + return grayName; } - - public void setUpdatedContent(String updatedContent) { - this.updatedContent = updatedContent; + + public void setGrayName(String grayName) { + this.grayName = grayName; } - + + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + public String getSrcIp() { return srcIp; } - + public void setSrcIp(String srcIp) { this.srcIp = srcIp; } - + public String getSrcUser() { return srcUser; } - + public void setSrcUser(String srcUser) { this.srcUser = srcUser; } - - public String getOpType() { - return opType; + + public String getOriginalMd5() { + return originalMd5; } - - public void setOpType(String opType) { - this.opType = opType; + + public void setOriginalMd5(String originalMd5) { + this.originalMd5 = originalMd5; } - - public String getPublishType() { - return publishType; + + public String getOriginalContent() { + return originalContent; } - - public void setPublishType(String publishType) { - this.publishType = publishType; + + public void setOriginalContent(String originalContent) { + this.originalContent = originalContent; } - - public String getExtInfo() { - return extInfo; + + public String getOriginalEncryptedDataKey() { + return originalEncryptedDataKey; } - - public void setExtInfo(String extInfo) { - this.extInfo = extInfo; + + public void setOriginalEncryptedDataKey(String originalEncryptedDataKey) { + this.originalEncryptedDataKey = originalEncryptedDataKey; } - - public Timestamp getCreatedTime() { - return new Timestamp(createdTime.getTime()); + + public String getOriginalExtInfo() { + return originalExtInfo; } - - public void setCreatedTime(Timestamp createdTime) { - this.createdTime = new Timestamp(createdTime.getTime()); + + public void setOriginalExtInfo(String originalExtInfo) { + this.originalExtInfo = originalExtInfo; } - - public Timestamp getLastModifiedTime() { - return new Timestamp(lastModifiedTime.getTime()); + + public String getUpdatedMd5() { + return updatedMd5; } - - public void setLastModifiedTime(Timestamp lastModifiedTime) { - this.lastModifiedTime = new Timestamp(lastModifiedTime.getTime()); + + public void setUpdatedMd5(String updatedMd5) { + this.updatedMd5 = updatedMd5; } - - public String getGrayName() { - return grayName; + + public String getUpdatedContent() { + return updatedContent; } - - public void setGrayName(String grayName) { - this.grayName = grayName; + + public void setUpdatedContent(String updatedContent) { + this.updatedContent = updatedContent; } - - public String getAppName() { - return appName; + + public String getUpdatedEncryptedDataKey() { + return updatedEncryptedDataKey; } - - public void setAppName(String appName) { - this.appName = appName; + + public void setUpdatedEncryptedDataKey(String updatedEncryptedDataKey) { + this.updatedEncryptedDataKey = updatedEncryptedDataKey; } - - public String getMd5() { - return md5; + + public String getUpdateExtInfo() { + return updateExtInfo; } - - public void setMd5(String md5) { - this.md5 = md5; + + public void setUpdateExtInfo(String updateExtInfo) { + this.updateExtInfo = updateExtInfo; } - - public String getEncryptedDataKey() { - return encryptedDataKey; + + public Timestamp getCreatedTime() { + return createdTime; } - - public void setEncryptedDataKey(String encryptedDataKey) { - this.encryptedDataKey = encryptedDataKey; + + public void setCreatedTime(Timestamp createdTime) { + this.createdTime = createdTime; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConfigHistoryInfoPair that = (ConfigHistoryInfoPair) o; - return id == that.id && lastId == that.lastId && Objects.equals(dataId, that.dataId) && Objects.equals(group, - that.group) && Objects.equals(tenant, that.tenant) && Objects.equals(appName, that.appName) - && Objects.equals(md5, that.md5) && Objects.equals(content, that.content) && Objects.equals(updatedMd5, - that.updatedMd5) && Objects.equals(updatedContent, that.updatedContent) && Objects.equals(srcIp, - that.srcIp) && Objects.equals(srcUser, that.srcUser) && Objects.equals(opType, that.opType) - && Objects.equals(createdTime, that.createdTime) && Objects.equals(lastModifiedTime, - that.lastModifiedTime) && Objects.equals(encryptedDataKey, that.encryptedDataKey); + + public Timestamp getLastModifiedTime() { + return lastModifiedTime; } - - @Override - public int hashCode() { - return Objects.hash(id, lastId, dataId, group, tenant, appName, md5, content, updatedMd5, updatedContent, srcIp, - srcUser, opType, createdTime, lastModifiedTime, encryptedDataKey); + + public void setLastModifiedTime(Timestamp lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; } - } \ No newline at end of file diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigGrayModelMigrateService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigGrayModelMigrateService.java index 22efb3422d..431661b81b 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigGrayModelMigrateService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigGrayModelMigrateService.java @@ -17,6 +17,7 @@ package com.alibaba.nacos.config.server.service; import com.alibaba.nacos.api.utils.NetUtils; +import com.alibaba.nacos.common.utils.ThreadUtils; import com.alibaba.nacos.config.server.model.ConfigInfoBetaWrapper; import com.alibaba.nacos.config.server.model.ConfigInfoGrayWrapper; import com.alibaba.nacos.config.server.model.ConfigInfoTagWrapper; @@ -29,6 +30,7 @@ import com.alibaba.nacos.config.server.service.repository.ConfigInfoTagPersistService; import com.alibaba.nacos.config.server.utils.PropertyUtil; import com.alibaba.nacos.persistence.model.Page; +import com.alibaba.nacos.sys.env.EnvUtil; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; @@ -142,10 +144,11 @@ public void checkMigrateTag(String dataId, String group, String tenant, String t private void doCheckMigrate() throws Exception { - ThreadPoolExecutor executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), - Runtime.getRuntime().availableProcessors(), 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(PropertyUtil.getAllDumpPageSize() * 2), - r -> new Thread(r, "gray migrate worker"), new ThreadPoolExecutor.CallerRunsPolicy()); + int migrateMulti = EnvUtil.getProperty("nacos.gray.migrate.executor.multi", Integer.class, Integer.valueOf(4)); + ThreadPoolExecutor executorService = new ThreadPoolExecutor(ThreadUtils.getSuitableThreadCount(migrateMulti), + ThreadUtils.getSuitableThreadCount(migrateMulti), 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(PropertyUtil.getAllDumpPageSize() * migrateMulti), + r -> new Thread(r, "gray-migrate-worker"), new ThreadPoolExecutor.CallerRunsPolicy()); int pageSize = 100; int rowCount = configInfoBetaPersistService.configInfoBetaCount(); int pageCount = (int) Math.ceil(rowCount * 1.0 / pageSize); diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/HistoryService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/HistoryService.java index e9fd5b7a80..1db81b932d 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/HistoryService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/HistoryService.java @@ -20,13 +20,13 @@ import com.alibaba.nacos.common.utils.StringUtils; import com.alibaba.nacos.config.server.enums.OperationType; import com.alibaba.nacos.config.server.model.ConfigHistoryInfo; -import com.alibaba.nacos.config.server.model.ConfigHistoryInfoPair; +import com.alibaba.nacos.config.server.model.ConfigHistoryInfoDetail; import com.alibaba.nacos.config.server.model.ConfigInfo; import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService; -import com.alibaba.nacos.persistence.model.Page; import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService; import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService; +import com.alibaba.nacos.persistence.model.Page; import com.alibaba.nacos.plugin.auth.exception.AccessException; import com.alibaba.nacos.plugin.encryption.handler.EncryptionHandler; import org.springframework.beans.BeanUtils; @@ -47,11 +47,12 @@ public class HistoryService { private final HistoryConfigInfoPersistService historyConfigInfoPersistService; private final ConfigInfoPersistService configInfoPersistService; - + private final ConfigInfoGrayPersistService configInfoGrayPersistService; public HistoryService(HistoryConfigInfoPersistService historyConfigInfoPersistService, - ConfigInfoPersistService configInfoPersistService, ConfigInfoGrayPersistService configInfoGrayPersistService) { + ConfigInfoPersistService configInfoPersistService, + ConfigInfoGrayPersistService configInfoGrayPersistService) { this.historyConfigInfoPersistService = historyConfigInfoPersistService; this.configInfoPersistService = configInfoPersistService; this.configInfoGrayPersistService = configInfoGrayPersistService; @@ -78,8 +79,8 @@ public ConfigHistoryInfo getConfigHistoryInfo(String dataId, String group, Strin checkHistoryInfoPermission(configHistoryInfo, dataId, group, namespaceId); String encryptedDataKey = configHistoryInfo.getEncryptedDataKey(); - Pair pair = EncryptionHandler - .decryptHandler(dataId, encryptedDataKey, configHistoryInfo.getContent()); + Pair pair = EncryptionHandler.decryptHandler(dataId, encryptedDataKey, + configHistoryInfo.getContent()); configHistoryInfo.setContent(pair.getSecond()); return configHistoryInfo; @@ -98,8 +99,8 @@ public ConfigHistoryInfo getPreviousConfigHistoryInfo(String dataId, String grou checkHistoryInfoPermission(configHistoryInfo, dataId, group, namespaceId); String encryptedDataKey = configHistoryInfo.getEncryptedDataKey(); - Pair pair = EncryptionHandler - .decryptHandler(dataId, encryptedDataKey, configHistoryInfo.getContent()); + Pair pair = EncryptionHandler.decryptHandler(dataId, encryptedDataKey, + configHistoryInfo.getContent()); configHistoryInfo.setContent(pair.getSecond()); return configHistoryInfo; @@ -117,70 +118,101 @@ public List getConfigListByNamespace(String namespaceId) { */ private void checkHistoryInfoPermission(ConfigHistoryInfo configHistoryInfo, String dataId, String group, String namespaceId) throws AccessException { - if (!Objects.equals(configHistoryInfo.getDataId(), dataId) || !Objects - .equals(configHistoryInfo.getGroup(), group) || !Objects - .equals(configHistoryInfo.getTenant(), namespaceId)) { + if (!Objects.equals(configHistoryInfo.getDataId(), dataId) || !Objects.equals(configHistoryInfo.getGroup(), + group) || !Objects.equals(configHistoryInfo.getTenant(), namespaceId)) { throw new AccessException("Please check dataId, group or namespaceId."); } } - + /** * Query the detailed config history info pair, including the original version and the updated version. */ - public ConfigHistoryInfoPair getConfigHistoryInfoPair(String dataId, String group, String namespaceId, Long nid) + public ConfigHistoryInfoDetail getConfigHistoryInfoDetail(String dataId, String group, String namespaceId, Long nid) throws AccessException { ConfigHistoryInfo configHistoryInfo = historyConfigInfoPersistService.detailConfigHistory(nid); if (Objects.isNull(configHistoryInfo)) { return null; } - + // check if history config match the input checkHistoryInfoPermission(configHistoryInfo, dataId, group, namespaceId); - + // transform - ConfigHistoryInfoPair configHistoryInfoPair = new ConfigHistoryInfoPair(); - BeanUtils.copyProperties(configHistoryInfo, configHistoryInfoPair); - configHistoryInfoPair.setOpType(configHistoryInfoPair.getOpType().trim()); - - if (OperationType.INSERT.getValue().equals(configHistoryInfoPair.getOpType())) { - configHistoryInfoPair.setUpdatedContent(configHistoryInfoPair.getContent()); - configHistoryInfoPair.setUpdatedMd5(configHistoryInfoPair.getMd5()); - configHistoryInfoPair.setContent(StringUtils.EMPTY); - configHistoryInfoPair.setMd5(StringUtils.EMPTY); + ConfigHistoryInfoDetail configHistoryInfoDetail = new ConfigHistoryInfoDetail(); + BeanUtils.copyProperties(configHistoryInfo, configHistoryInfoDetail); + configHistoryInfoDetail.setOpType(configHistoryInfoDetail.getOpType().trim()); + + //insert + if (OperationType.INSERT.getValue().equals(configHistoryInfoDetail.getOpType())) { + configHistoryInfoDetail.setUpdatedContent(configHistoryInfo.getContent()); + configHistoryInfoDetail.setUpdatedMd5(configHistoryInfo.getMd5()); + configHistoryInfoDetail.setUpdatedEncryptedDataKey(configHistoryInfo.getEncryptedDataKey()); + configHistoryInfoDetail.setUpdateExtInfo(configHistoryInfo.getExtInfo()); + configHistoryInfoDetail.setOriginalExtInfo(StringUtils.EMPTY); + configHistoryInfoDetail.setOriginalContent(StringUtils.EMPTY); + configHistoryInfoDetail.setOriginalMd5(StringUtils.EMPTY); + configHistoryInfoDetail.setOriginalEncryptedDataKey(StringUtils.EMPTY); } - - if (OperationType.UPDATE.getValue().equals(configHistoryInfoPair.getOpType())) { + + //update + if (OperationType.UPDATE.getValue().equals(configHistoryInfoDetail.getOpType())) { + + configHistoryInfoDetail.setOriginalExtInfo(configHistoryInfo.getExtInfo()); + configHistoryInfoDetail.setOriginalContent(configHistoryInfo.getContent()); + configHistoryInfoDetail.setOriginalMd5(configHistoryInfo.getMd5()); + configHistoryInfoDetail.setOriginalEncryptedDataKey(configHistoryInfo.getEncryptedDataKey()); + ConfigHistoryInfo nextHistoryInfo = historyConfigInfoPersistService.getNextHistoryInfo(dataId, group, - namespaceId, configHistoryInfoPair.getPublishType(), configHistoryInfoPair.getGrayName(), nid); + namespaceId, configHistoryInfoDetail.getPublishType(), configHistoryInfoDetail.getGrayName(), nid); + + ConfigInfo currentConfigInfo = null; if (Objects.isNull(nextHistoryInfo)) { - // get the latest config info - ConfigInfo configInfo = StringUtils.isEmpty(configHistoryInfoPair.getGrayName()) + //double check for concurrent + currentConfigInfo = StringUtils.isEmpty(configHistoryInfoDetail.getGrayName()) ? configInfoPersistService.findConfigInfo(dataId, group, namespaceId) : configInfoGrayPersistService.findConfigInfo4Gray(dataId, group, namespaceId, - configHistoryInfoPair.getGrayName()); - configHistoryInfoPair.setUpdatedMd5(configInfo.getMd5()); - configHistoryInfoPair.setUpdatedContent(configInfo.getContent()); + configHistoryInfoDetail.getGrayName()); + nextHistoryInfo = historyConfigInfoPersistService.getNextHistoryInfo(dataId, group, namespaceId, + configHistoryInfoDetail.getPublishType(), configHistoryInfoDetail.getGrayName(), nid); + + } + + if (nextHistoryInfo != null) { + configHistoryInfoDetail.setUpdateExtInfo(nextHistoryInfo.getExtInfo()); + configHistoryInfoDetail.setUpdatedContent(nextHistoryInfo.getContent()); + configHistoryInfoDetail.setUpdatedMd5(nextHistoryInfo.getMd5()); + configHistoryInfoDetail.setUpdatedEncryptedDataKey(nextHistoryInfo.getEncryptedDataKey()); } else { - configHistoryInfoPair.setUpdatedMd5(nextHistoryInfo.getMd5()); - configHistoryInfoPair.setUpdatedContent(nextHistoryInfo.getContent()); + configHistoryInfoDetail.setUpdatedContent(currentConfigInfo.getContent()); + configHistoryInfoDetail.setUpdatedMd5(currentConfigInfo.getMd5()); + configHistoryInfoDetail.setUpdatedEncryptedDataKey(currentConfigInfo.getEncryptedDataKey()); + } } - - if (OperationType.DELETE.getValue().equals(configHistoryInfoPair.getOpType())) { - configHistoryInfoPair.setUpdatedMd5(StringUtils.EMPTY); - configHistoryInfoPair.setUpdatedContent(StringUtils.EMPTY); + + //delete + if (OperationType.DELETE.getValue().equals(configHistoryInfoDetail.getOpType())) { + configHistoryInfoDetail.setOriginalMd5(configHistoryInfo.getMd5()); + configHistoryInfoDetail.setOriginalContent(configHistoryInfo.getContent()); + configHistoryInfoDetail.setOriginalEncryptedDataKey(configHistoryInfo.getEncryptedDataKey()); + configHistoryInfoDetail.setOriginalExtInfo(configHistoryInfo.getExtInfo()); } - + // decrypt content - String encryptedDataKey = configHistoryInfoPair.getEncryptedDataKey(); - String originalContent = EncryptionHandler.decryptHandler(dataId, encryptedDataKey, - configHistoryInfoPair.getContent()).getSecond(); - configHistoryInfoPair.setContent(originalContent); - - String updatedContent = EncryptionHandler.decryptHandler(dataId, encryptedDataKey, - configHistoryInfoPair.getUpdatedContent()).getSecond(); - configHistoryInfoPair.setUpdatedContent(updatedContent); - - return configHistoryInfoPair; + if (StringUtils.isNotBlank(configHistoryInfoDetail.getOriginalContent())) { + String originalContent = EncryptionHandler.decryptHandler(dataId, + configHistoryInfoDetail.getOriginalEncryptedDataKey(), configHistoryInfoDetail.getOriginalContent()) + .getSecond(); + configHistoryInfoDetail.setOriginalContent(originalContent); + } + if (StringUtils.isNotBlank(configHistoryInfoDetail.getUpdatedContent())) { + String updatedContent = EncryptionHandler.decryptHandler(dataId, + configHistoryInfoDetail.getUpdatedEncryptedDataKey(), configHistoryInfoDetail.getUpdatedContent()) + .getSecond(); + configHistoryInfoDetail.setUpdatedContent(updatedContent); + } + + return configHistoryInfoDetail; } + } \ No newline at end of file diff --git a/config/src/test/java/com/alibaba/nacos/config/server/controller/HistoryControllerTest.java b/config/src/test/java/com/alibaba/nacos/config/server/controller/HistoryControllerTest.java index b0e8e58b79..7b694f58f6 100644 --- a/config/src/test/java/com/alibaba/nacos/config/server/controller/HistoryControllerTest.java +++ b/config/src/test/java/com/alibaba/nacos/config/server/controller/HistoryControllerTest.java @@ -17,10 +17,8 @@ package com.alibaba.nacos.config.server.controller; import com.alibaba.nacos.common.utils.JacksonUtils; -import com.alibaba.nacos.common.utils.StringUtils; import com.alibaba.nacos.config.server.constant.Constants; import com.alibaba.nacos.config.server.model.ConfigHistoryInfo; -import com.alibaba.nacos.config.server.model.ConfigHistoryInfoPair; import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; import com.alibaba.nacos.config.server.service.HistoryService; import com.alibaba.nacos.persistence.model.Page; @@ -96,9 +94,9 @@ void testListConfigHistory() throws Exception { when(historyService.listConfigHistory("test", "test", "", 1, 10)).thenReturn(page); - MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH).param("search", "accurate") - .param("dataId", "test").param("group", "test").param("tenant", "").param("appName", "").param("pageNo", "1") - .param("pageSize", "10"); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH) + .param("search", "accurate").param("dataId", "test").param("group", "test").param("tenant", "") + .param("appName", "").param("pageNo", "1").param("pageSize", "10"); MockHttpServletResponse response = mockmvc.perform(builder).andReturn().getResponse(); @@ -106,7 +104,8 @@ void testListConfigHistory() throws Exception { JsonNode pageItemsNode = JacksonUtils.toObj(actualValue).get("pageItems"); List resultList = JacksonUtils.toObj(pageItemsNode.toString(), List.class); - ConfigHistoryInfo resConfigHistoryInfo = JacksonUtils.toObj(pageItemsNode.get(0).toString(), ConfigHistoryInfo.class); + ConfigHistoryInfo resConfigHistoryInfo = JacksonUtils.toObj(pageItemsNode.get(0).toString(), + ConfigHistoryInfo.class); assertEquals(configHistoryInfoList.size(), resultList.size()); assertEquals(configHistoryInfo.getDataId(), resConfigHistoryInfo.getDataId()); @@ -128,8 +127,8 @@ void testGetConfigHistoryInfo() throws Exception { when(historyController.getConfigHistoryInfo("test", "test", "", 1L)).thenReturn(configHistoryInfo); - MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH).param("dataId", "test") - .param("group", "test").param("tenant", "").param("nid", "1"); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH) + .param("dataId", "test").param("group", "test").param("tenant", "").param("nid", "1"); String actualValue = mockmvc.perform(builder).andReturn().getResponse().getContentAsString(); ConfigHistoryInfo resConfigHistoryInfo = JacksonUtils.toObj(actualValue, ConfigHistoryInfo.class); @@ -153,8 +152,9 @@ void testGetPreviousConfigHistoryInfo() throws Exception { when(historyService.getPreviousConfigHistoryInfo("test", "test", "", 1L)).thenReturn(configHistoryInfo); - MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH + "/previous") - .param("dataId", "test").param("group", "test").param("tenant", "").param("id", "1"); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get( + Constants.HISTORY_CONTROLLER_PATH + "/previous").param("dataId", "test").param("group", "test") + .param("tenant", "").param("id", "1"); String actualValue = mockmvc.perform(builder).andReturn().getResponse().getContentAsString(); ConfigHistoryInfo resConfigHistoryInfo = JacksonUtils.toObj(actualValue, ConfigHistoryInfo.class); @@ -176,8 +176,8 @@ void testGetDataIds() throws Exception { configInfoWrappers.add(configInfoWrapper); when(historyService.getConfigListByNamespace("test")).thenReturn(configInfoWrappers); - MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH + "/configs") - .param("tenant", "test"); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get( + Constants.HISTORY_CONTROLLER_PATH + "/configs").param("tenant", "test"); String actualValue = mockmvc.perform(builder).andReturn().getResponse().getContentAsString(); List resConfigInfoWrappers = JacksonUtils.toObj(actualValue, List.class); @@ -190,33 +190,5 @@ void testGetDataIds() throws Exception { assertEquals(configInfoWrapper.getContent(), resConfigInfoWrapper.getContent()); } - - @Test - void testGetConfigHistoryInfoPair() throws Exception { - - ConfigHistoryInfoPair configHistoryInfoPair = new ConfigHistoryInfoPair(); - configHistoryInfoPair.setDataId("test"); - configHistoryInfoPair.setGroup("test"); - configHistoryInfoPair.setContent("test"); - configHistoryInfoPair.setUpdatedContent("test updated"); - configHistoryInfoPair.setTenant(StringUtils.EMPTY); - configHistoryInfoPair.setCreatedTime(new Timestamp(new Date().getTime())); - configHistoryInfoPair.setLastModifiedTime(new Timestamp(new Date().getTime())); - - when(historyController.getConfigHistoryInfoPair("test", "test", StringUtils.EMPTY, 1L)) - .thenReturn(configHistoryInfoPair); - - MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.get(Constants.HISTORY_CONTROLLER_PATH + "/pair") - .param("dataId", "test").param("group", "test") - .param("tenant", "").param("nid", "1"); - - String actualValue = mockmvc.perform(builder).andReturn().getResponse().getContentAsString(); - ConfigHistoryInfoPair resConfigHistoryInfoPair = JacksonUtils.toObj(actualValue, ConfigHistoryInfoPair.class); - - assertEquals(configHistoryInfoPair.getDataId(), resConfigHistoryInfoPair.getDataId()); - assertEquals(configHistoryInfoPair.getGroup(), resConfigHistoryInfoPair.getGroup()); - assertEquals(configHistoryInfoPair.getContent(), resConfigHistoryInfoPair.getContent()); - assertEquals(configHistoryInfoPair.getUpdatedContent(), resConfigHistoryInfoPair.getUpdatedContent()); - } - + } diff --git a/config/src/test/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2Test.java b/config/src/test/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2Test.java index df6c061359..3bb58a306d 100644 --- a/config/src/test/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2Test.java +++ b/config/src/test/java/com/alibaba/nacos/config/server/controller/v2/HistoryControllerV2Test.java @@ -89,8 +89,8 @@ void testListConfigHistory() throws Exception { when(historyService.listConfigHistory(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1, 10)).thenReturn(page); - Result> pageResult = historyControllerV2.listConfigHistory(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1, - 10); + Result> pageResult = historyControllerV2.listConfigHistory(TEST_DATA_ID, TEST_GROUP, + TEST_NAMESPACE_ID, 1, 10); verify(historyService).listConfigHistory(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1, 10); @@ -152,9 +152,11 @@ void testGetConfigHistoryInfoWhenNameSpaceIsPublic() throws Exception { configHistoryInfo.setCreatedTime(new Timestamp(new Date().getTime())); configHistoryInfo.setLastModifiedTime(new Timestamp(new Date().getTime())); - when(historyService.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn(configHistoryInfo); + when(historyService.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn( + configHistoryInfo); - Result result = historyControllerV2.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID_PUBLIC, 1L); + Result result = historyControllerV2.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, + TEST_NAMESPACE_ID_PUBLIC, 1L); verify(historyService).getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L); @@ -178,9 +180,11 @@ void testGetConfigHistoryInfo() throws Exception { configHistoryInfo.setCreatedTime(new Timestamp(new Date().getTime())); configHistoryInfo.setLastModifiedTime(new Timestamp(new Date().getTime())); - when(historyService.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn(configHistoryInfo); + when(historyService.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn( + configHistoryInfo); - Result result = historyControllerV2.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L); + Result result = historyControllerV2.getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, + TEST_NAMESPACE_ID, 1L); verify(historyService).getConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L); @@ -204,10 +208,11 @@ void testGetPreviousConfigHistoryInfo() throws Exception { configHistoryInfo.setCreatedTime(new Timestamp(new Date().getTime())); configHistoryInfo.setLastModifiedTime(new Timestamp(new Date().getTime())); - when(historyService.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn(configHistoryInfo); + when(historyService.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn( + configHistoryInfo); - Result result = historyControllerV2.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, - 1L); + Result result = historyControllerV2.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, + TEST_NAMESPACE_ID, 1L); verify(historyService).getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L); @@ -231,7 +236,8 @@ void testGetPreviousConfigHistoryInfoWhenNameSpaceIsPublic() throws Exception { configHistoryInfo.setCreatedTime(new Timestamp(new Date().getTime())); configHistoryInfo.setLastModifiedTime(new Timestamp(new Date().getTime())); - when(historyService.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn(configHistoryInfo); + when(historyService.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID, 1L)).thenReturn( + configHistoryInfo); Result result = historyControllerV2.getPreviousConfigHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_NAMESPACE_ID_PUBLIC, 1L); @@ -288,4 +294,5 @@ void testGetConfigListByNamespaceWhenIsPublic() throws NacosApiException { assertEquals(configInfoWrapper.getGroup(), actualConfigInfoWrapper.getGroup()); assertEquals(configInfoWrapper.getContent(), actualConfigInfoWrapper.getContent()); } + } diff --git a/config/src/test/java/com/alibaba/nacos/config/server/service/HistoryServiceTest.java b/config/src/test/java/com/alibaba/nacos/config/server/service/HistoryServiceTest.java index 0ebbfdcd50..c813392d5b 100644 --- a/config/src/test/java/com/alibaba/nacos/config/server/service/HistoryServiceTest.java +++ b/config/src/test/java/com/alibaba/nacos/config/server/service/HistoryServiceTest.java @@ -20,7 +20,7 @@ import com.alibaba.nacos.config.server.constant.Constants; import com.alibaba.nacos.config.server.enums.OperationType; import com.alibaba.nacos.config.server.model.ConfigHistoryInfo; -import com.alibaba.nacos.config.server.model.ConfigHistoryInfoPair; +import com.alibaba.nacos.config.server.model.ConfigHistoryInfoDetail; import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService; import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService; @@ -214,15 +214,15 @@ void testGetConfigHistoryInfoPair() throws Exception { when(historyConfigInfoPersistService.getNextHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_TENANT, Constants.FORMAL, StringUtils.EMPTY, 1L)).thenReturn(nextHistoryInfo); - ConfigHistoryInfoPair resConfigHistoryInfoPair = historyService.getConfigHistoryInfoPair(TEST_DATA_ID, TEST_GROUP, + ConfigHistoryInfoDetail resConfigHistoryInfoDetail = historyService.getConfigHistoryInfoDetail(TEST_DATA_ID, TEST_GROUP, TEST_TENANT, 1L); verify(historyConfigInfoPersistService).getNextHistoryInfo(TEST_DATA_ID, TEST_GROUP, TEST_TENANT, Constants.FORMAL, StringUtils.EMPTY, 1L); - assertEquals(nextHistoryInfo.getDataId(), resConfigHistoryInfoPair.getDataId()); - assertEquals(nextHistoryInfo.getGroup(), resConfigHistoryInfoPair.getGroup()); - assertEquals(nextHistoryInfo.getMd5(), resConfigHistoryInfoPair.getUpdatedMd5()); - assertEquals(nextHistoryInfo.getContent(), resConfigHistoryInfoPair.getUpdatedContent()); + assertEquals(nextHistoryInfo.getDataId(), resConfigHistoryInfoDetail.getDataId()); + assertEquals(nextHistoryInfo.getGroup(), resConfigHistoryInfoDetail.getGroup()); + assertEquals(nextHistoryInfo.getMd5(), resConfigHistoryInfoDetail.getUpdatedMd5()); + assertEquals(nextHistoryInfo.getContent(), resConfigHistoryInfoDetail.getUpdatedContent()); } } diff --git a/console/src/main/resources/application.properties b/console/src/main/resources/application.properties index a3629d9118..5cb987322d 100644 --- a/console/src/main/resources/application.properties +++ b/console/src/main/resources/application.properties @@ -33,14 +33,14 @@ server.port=8848 ### Deprecated configuration property, it is recommended to use `spring.sql.init.platform` replaced. # spring.datasource.platform=mysql # nacos.plugin.datasource.log.enabled=true -#spring.sql.init.platform=mysql +spring.sql.init.platform=mysql ### Count of DB: -# db.num=1 +db.num=1 ### Connect URL of DB: -# db.url.0=jdbc:mysql://127.0.0.1:3306/nacos?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC -# db.user=nacos -# db.password=nacos +db.url.0=jdbc:mysql://127.0.0.1:3306/nacos?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC +db.user=root +db.password=11021102 ### the maximum retry times for push nacos.config.push.maxRetryTime=50 diff --git a/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/mapper/HistoryConfigInfoMapper.java b/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/mapper/HistoryConfigInfoMapper.java index 32ddd63716..96dd13d060 100644 --- a/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/mapper/HistoryConfigInfoMapper.java +++ b/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/mapper/HistoryConfigInfoMapper.java @@ -137,7 +137,7 @@ default MapperResult getNextHistoryInfo(MapperContext context) { String sql = "SELECT nid,data_id,group_id,tenant_id,app_name,content,md5,src_user,src_ip,op_type,publish_type," + "gray_name,ext_info,gmt_create,gmt_modified,encrypted_data_key FROM his_config_info " + "WHERE data_id = ? AND group_id = ? AND tenant_id = ? AND publish_type = ? " - + (StringUtils.isEmpty(context.getContextParameter(FieldConstant.GRAY_NAME)) ? "" : "AND gray_name = ? ") + + (StringUtils.isBlank(context.getContextParameter(FieldConstant.GRAY_NAME)) ? "" : "AND gray_name = ? ") + "AND nid > ? ORDER BY nid LIMIT 1"; List paramList = CollectionUtils.list(