From dce8ef00c5bee2a6a44f5f2419100876d15f205a Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Wed, 29 Jun 2022 16:00:29 +0800 Subject: [PATCH 01/84] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=B5=81=E7=A8=8B-?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=9A=84=E6=97=B6=E5=80=99?= =?UTF-8?q?=EF=BC=8C=E5=A6=82=E6=9E=9C=E6=9C=89=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E5=BF=AB=E9=80=9F=E5=A4=B1=E8=B4=A5=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/diff/api/DiffFileApi.java | 9 +++ .../web/diff/cloud/impl/DiffFileApiImpl.java | 67 +++++++++++++++---- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/takin-web-service-adapters/takin-web-diff/takin-web-diff-api/src/main/java/io/shulie/takin/web/diff/api/DiffFileApi.java b/takin-web-service-adapters/takin-web-diff/takin-web-diff-api/src/main/java/io/shulie/takin/web/diff/api/DiffFileApi.java index 9c3deb0eb5..92f580b55f 100644 --- a/takin-web-service-adapters/takin-web-diff/takin-web-diff-api/src/main/java/io/shulie/takin/web/diff/api/DiffFileApi.java +++ b/takin-web-service-adapters/takin-web-diff/takin-web-diff-api/src/main/java/io/shulie/takin/web/diff/api/DiffFileApi.java @@ -6,6 +6,8 @@ import io.shulie.takin.cloud.sdk.model.request.filemanager.FileDeleteParamReq; import io.shulie.takin.cloud.sdk.model.request.filemanager.FileZipParamReq; +import java.util.List; + /** * @author shiyajian * create: 2020-10-19 @@ -46,4 +48,11 @@ public interface DiffFileApi { * @return 文件的MD5值 */ String createFileByPathAndString(FileCreateByStringParamReq req); + + /** + * 判断文件是否存在,这里只校验file,不管path + * @param pathList + * @return + */ + boolean isExist(List pathList); } diff --git a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java index 7ad9b67b64..5510b2eed9 100644 --- a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java +++ b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java @@ -1,21 +1,22 @@ package io.shulie.takin.web.diff.cloud.impl; -import java.util.Map; - import io.shulie.takin.cloud.entrypoint.file.CloudFileApi; -import io.shulie.takin.cloud.sdk.model.request.filemanager.FileContentParamReq; -import io.shulie.takin.cloud.sdk.model.request.filemanager.FileCopyParamReq; -import io.shulie.takin.cloud.sdk.model.request.filemanager.FileCreateByStringParamReq; -import io.shulie.takin.cloud.sdk.model.request.filemanager.FileDeleteParamReq; -import io.shulie.takin.cloud.sdk.model.request.filemanager.FileZipParamReq; -import io.shulie.takin.common.beans.response.ResponseResult; +import io.shulie.takin.cloud.sdk.model.request.filemanager.*; import io.shulie.takin.utils.string.StringUtil; +import io.shulie.takin.web.common.exception.TakinWebException; +import io.shulie.takin.web.common.exception.TakinWebExceptionEnum; import io.shulie.takin.web.diff.api.DiffFileApi; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.io.File; +import java.util.List; +import java.util.Map; + /** * @author shiyajian * create: 2020-10-19 @@ -26,8 +27,10 @@ public class DiffFileApiImpl implements DiffFileApi { @Autowired private CloudFileApi cloudFileApi; - @Override + @Value("${file.upload.openFileValid:true}") + private boolean openFileValid; + @Override public String getFileManageContextPath(FileContentParamReq req) { Map fileContent = cloudFileApi.getFileContent(req); if (fileContent != null && fileContent.containsKey(String.join(",", req.getPaths()))) { @@ -38,7 +41,9 @@ public String getFileManageContextPath(FileContentParamReq req) { @Override public Boolean deleteFile(FileDeleteParamReq req) { - if (CollectionUtils.isEmpty(req.getPaths())) {return true;} + if (CollectionUtils.isEmpty(req.getPaths())) { + return true; + } try { return cloudFileApi.deleteFile(req); } catch (Exception e) { @@ -49,18 +54,25 @@ public Boolean deleteFile(FileDeleteParamReq req) { @Override public Boolean copyFile(FileCopyParamReq req) { if (CollectionUtils.isEmpty(req.getSourcePaths()) - || StringUtil.isBlank(req.getTargetPath())) {return true;} + || StringUtil.isBlank(req.getTargetPath())) { + return true; + } try { + if (!this.isExist(req.getSourcePaths())) { + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件校验失败！"); + } return cloudFileApi.copyFile(req); } catch (Exception e) { - return false; + log.error("copyFile is fail ", ExceptionUtils.getStackTrace(e)); + // 这里直接抛异常处理,不能吃掉异常信息 + throw e; } } @Override public Boolean zipFile(FileZipParamReq req) { if (CollectionUtils.isEmpty(req.getSourcePaths()) || - StringUtil.isBlank(req.getTargetPath()) || StringUtil.isBlank(req.getZipFileName())) { + StringUtil.isBlank(req.getTargetPath()) || StringUtil.isBlank(req.getZipFileName())) { return false; } try { @@ -78,4 +90,33 @@ public String createFileByPathAndString(FileCreateByStringParamReq req) { return cloudFileApi.createFileByPathAndString(req); } + /** + * 批量验证文件 + * + * @param pathList + * @return + */ + @Override + public boolean isExist(List pathList) { + if (!openFileValid) { + // 关闭的时候直接返回成功 + return true; + } + if (CollectionUtils.isEmpty(pathList)) { + return false; + } + boolean isExist = true; + for (int i = 0; i < pathList.size(); i++) { + File file = new File(pathList.get(i)); + if (file != null && file.isFile()) { + // 继续验证下一个 + } else { + isExist = false; + log.error("复制的文件不存在,", pathList.get(i)); + break; + } + } + return isExist; + } + } From 23d581cf71a29c3c8656790eb49cb32f98477f04 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Wed, 29 Jun 2022 16:10:48 +0800 Subject: [PATCH 02/84] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=B5=81=E7=A8=8B-?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=9A=84=E6=97=B6=E5=80=99?= =?UTF-8?q?=EF=BC=8C=E5=A6=82=E6=9E=9C=E6=9C=89=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E5=BF=AB=E9=80=9F=E5=A4=B1=E8=B4=A5=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java index 5510b2eed9..78c4758c8d 100644 --- a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java +++ b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java @@ -59,9 +59,12 @@ public Boolean copyFile(FileCopyParamReq req) { } try { if (!this.isExist(req.getSourcePaths())) { - throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件校验失败！"); + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件校验失败！请重试！"); } - return cloudFileApi.copyFile(req); + if (!cloudFileApi.copyFile(req)) { + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件操作失败,请重试！"); + } + return true; } catch (Exception e) { log.error("copyFile is fail ", ExceptionUtils.getStackTrace(e)); // 这里直接抛异常处理,不能吃掉异常信息 From 460edd75493e25ea088ce6fc2fd44a161c456946 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Wed, 29 Jun 2022 17:06:12 +0800 Subject: [PATCH 03/84] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=B5=81=E7=A8=8B-?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=9A=84=E6=97=B6=E5=80=99?= =?UTF-8?q?=EF=BC=8C=E5=A6=82=E6=9E=9C=E6=9C=89=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E5=BF=AB=E9=80=9F=E5=A4=B1=E8=B4=A5=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java index 78c4758c8d..a0b3375922 100644 --- a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java +++ b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java @@ -1,5 +1,6 @@ package io.shulie.takin.web.diff.cloud.impl; +import com.alibaba.fastjson.JSON; import io.shulie.takin.cloud.entrypoint.file.CloudFileApi; import io.shulie.takin.cloud.sdk.model.request.filemanager.*; import io.shulie.takin.utils.string.StringUtil; @@ -59,7 +60,7 @@ public Boolean copyFile(FileCopyParamReq req) { } try { if (!this.isExist(req.getSourcePaths())) { - throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件校验失败！请重试！"); + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "关联的文件已丢失,请检查!!!"); } if (!cloudFileApi.copyFile(req)) { throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "文件操作失败,请重试！"); @@ -115,7 +116,7 @@ public boolean isExist(List pathList) { // 继续验证下一个 } else { isExist = false; - log.error("复制的文件不存在,", pathList.get(i)); + log.error("复制的文件不存在,请检查" + pathList.get(i)); break; } } From 25823768e7d0b55a9a9b2d525c5efbb4a2e87290 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Wed, 29 Jun 2022 17:09:21 +0800 Subject: [PATCH 04/84] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=B5=81=E7=A8=8B-?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E7=9A=84=E6=97=B6=E5=80=99?= =?UTF-8?q?=EF=BC=8C=E5=A6=82=E6=9E=9C=E6=9C=89=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E5=BF=AB=E9=80=9F=E5=A4=B1=E8=B4=A5=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java index a0b3375922..457c5f898f 100644 --- a/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java +++ b/takin-web-service-adapters/takin-web-diff/takin-web-diff-cloud/src/main/java/io/shulie/takin/web/diff/cloud/impl/DiffFileApiImpl.java @@ -67,7 +67,7 @@ public Boolean copyFile(FileCopyParamReq req) { } return true; } catch (Exception e) { - log.error("copyFile is fail ", ExceptionUtils.getStackTrace(e)); + log.error("copyFile is fail " + ExceptionUtils.getStackTrace(e)); // 这里直接抛异常处理,不能吃掉异常信息 throw e; } From 5fb30fcc238c17fbfc1b385750c3c83b50bca39f Mon Sep 17 00:00:00 2001 From: "pnz.zhao" Date: Fri, 1 Jul 2022 16:48:21 +0800 Subject: [PATCH 05/84] =?UTF-8?q?=E9=A1=BA=E4=B8=B02.0=E5=8E=8B=E6=B5=8B?= =?UTF-8?q?=E6=85=A2SQL=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application.properties | 5 +- .../impl/AppRemoteCallServiceImplTest.java | 64 ++++++++++++++----- .../impl/AppRemoteCallServiceImpl.java | 13 ++-- .../impl/AppRemoteCallDAOImpl.java | 22 ++++++- 4 files changed, 79 insertions(+), 25 deletions(-) diff --git a/takin-web-app/src/main/resources/application.properties b/takin-web-app/src/main/resources/application.properties index 750e806ac4..e295c12d44 100644 --- a/takin-web-app/src/main/resources/application.properties +++ b/takin-web-app/src/main/resources/application.properties @@ -141,4 +141,7 @@ takin.data.path=${data.path} takin.inner.pre=0 # 设置登录超时时间，默认24小时 -login.expire.time=86400 \ No newline at end of file +login.expire.time=86400 + +# 处理入口规则一次处理应用数 +takin.job.app.limit=10 \ No newline at end of file diff --git a/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java b/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java index f407b414c3..1f493bf79d 100644 --- a/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java +++ b/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java @@ -25,19 +25,26 @@ import io.shulie.takin.web.biz.pojo.request.datasource.DataSourceQueryRequest; import io.shulie.takin.web.biz.pojo.response.datasource.DatasourceListResponse; import io.shulie.takin.web.biz.service.DataSourceService; +import io.shulie.takin.web.biz.service.async.AsyncService; import io.shulie.takin.web.biz.service.linkmanage.AppRemoteCallService; import io.shulie.takin.web.common.enums.application.AppRemoteCallConfigEnum; +import io.shulie.takin.web.data.param.perfomanceanaly.PerformanceBaseDataParam; +import io.shulie.takin.web.data.param.perfomanceanaly.PerformanceThreadDataParam; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.security.core.parameters.P; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; +import java.util.ArrayList; +import java.util.List; + /** * 说明: * @@ -66,14 +73,37 @@ public class AppRemoteCallServiceImplTest { @Autowired DataSourceService dataSourceService; + @Autowired + AsyncService asyncService; + + @Test + public void zzTest(){ + PerformanceBaseDataParam param = new PerformanceBaseDataParam(); + param.setAgentId("11"); + param.setTimestamp(System.currentTimeMillis()); + List list = new ArrayList<>(); + for(int i=0;i<50;i++){ +// int i=9; + PerformanceThreadDataParam p1 = new PerformanceThreadDataParam(); + p1.setThreadStackLink(1L); + p1.setThreadId(i+"1"); + p1.setThreadName(i+"t1"); + p1.setGroupName(i+"g1"); + p1.setThreadCpuUsage(1D); + list.add(p1); + } + param.setThreadDataList(list); + asyncService.savePerformanceBaseData(param); + } + @Test public void batchConfigTest(){ - TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition); - AppRemoteCallConfigRequest request = new AppRemoteCallConfigRequest(); - request.setType(AppRemoteCallConfigEnum.OPEN_WHITELIST.getType().shortValue()); - request.setAppIds(Lists.newArrayList(6841250229157629952L,6846731729676275712L)); - appRemoteCallService.batchConfig(request); - dataSourceTransactionManager.commit(transactionStatus);//提交 +// TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition); +// AppRemoteCallConfigRequest request = new AppRemoteCallConfigRequest(); +// request.setType(AppRemoteCallConfigEnum.OPEN_WHITELIST.getType().shortValue()); +// request.setAppIds(Lists.newArrayList(6841250229157629952L,6846731729676275712L)); +// appRemoteCallService.batchConfig(request); +// dataSourceTransactionManager.commit(transactionStatus);//提交 } @Test @@ -92,15 +122,15 @@ public void agentTest(){ //System.out.println(completeSimulatorPath); } - @Test - public void like(){ - final DataSourceQueryRequest request = new DataSourceQueryRequest(); - request.setCurrent(0); - request.setPageSize(10); - //request.setDatasourceName("%"); - request.setJdbcUrl("_0"); - final PagingList list = dataSourceService.listDatasource( - request); - System.out.println(JSON.toJSONString(list)); - } +// @Test +// public void like(){ +// final DataSourceQueryRequest request = new DataSourceQueryRequest(); +// request.setCurrent(0); +// request.setPageSize(10); +// //request.setDatasourceName("%"); +// request.setJdbcUrl("_0"); +// final PagingList list = dataSourceService.listDatasource( +// request); +// System.out.println(JSON.toJSONString(list)); +// } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java index 5a8e6b9561..1bb3851ef8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java @@ -95,6 +95,7 @@ import org.apache.commons.lang3.StringUtils; import org.mockito.internal.util.collections.Sets; import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; /** @@ -131,6 +132,9 @@ public class AppRemoteCallServiceImpl implements AppRemoteCallService { @Resource private ThreadPoolExecutor queryAsyncThreadPool; + @Value("${takin.job.app.limit:50}") + private int appSize = 0; + @PostConstruct public void init() { criticaValue = ConfigServerHelper.getWrapperIntegerValueByKey( @@ -545,18 +549,17 @@ public void syncAmdb() { } List voList = dictionaryDataDAO.getDictByCode("REMOTE_CALL_TYPE"); - int size = 50; // size个轮询一次 - if (results.size() > size) { + if (results.size() > appSize) { int i = 1; boolean loop = true; do { List subList; //批量处理 - if (results.size() > i * size) { - subList = results.subList((i - 1) * size, i * size); + if (results.size() > i * appSize) { + subList = results.subList((i - 1) * appSize, i * appSize); } else { - subList = results.subList((i - 1) * size, results.size()); + subList = results.subList((i - 1) * appSize, results.size()); loop = false; } i++; diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java index e425f74aa3..5cbe2cdb1c 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java @@ -16,6 +16,7 @@ package io.shulie.takin.web.data.dao.application.impl; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -209,7 +210,7 @@ private LambdaQueryWrapper getAppRemoteCallEntityLambdaQuer if (param.getIsSynchronize() != null) { lambdaQueryWrapper.eq(AppRemoteCallEntity::getIsSynchronize, param.getIsSynchronize()); } - lambdaQueryWrapper.orderByDesc(AppRemoteCallEntity::getGmtModified); + //lambdaQueryWrapper.orderByDesc(AppRemoteCallEntity::getGmtModified);优化慢sql return lambdaQueryWrapper; } @@ -309,7 +310,24 @@ public List getAllRecord() { LambdaQueryWrapper lambdaQueryWrapper = this.getLambdaQueryWrapper() .eq(AppRemoteCallEntity::getIsDeleted, 0); - List list = list(lambdaQueryWrapper); + //改分页查,优化慢sql + List list = new ArrayList<>(); + List listTemp; + int i=1; + boolean doFlag; + do { + Page page = new Page<>(i, 10000); + IPage entityPageInfo = this.page(page, lambdaQueryWrapper); + if (CollectionUtils.isNotEmpty(entityPageInfo.getRecords())) { + doFlag = true; + listTemp = entityPageInfo.getRecords(); + list.addAll(listTemp); + i++; + }else{ + doFlag = false; + } + }while (doFlag); + if (list.isEmpty()) { return Collections.emptyList(); } From 04cdbd8db3c8b0648bac67d5ac2f820275d97948 Mon Sep 17 00:00:00 2001 From: nanfeng Date: Mon, 4 Jul 2022 16:53:56 +0800 Subject: [PATCH 06/84] =?UTF-8?q?=E5=9B=A0=E4=B8=BA=E6=9F=90=E4=BA=9B?= =?UTF-8?q?=E6=9C=AA=E7=9F=A5=E5=8E=9F=E5=9B=A0=E5=AF=BC=E8=87=B4=E5=81=B6?= =?UTF-8?q?=E5=B0=94=E9=87=8D=E5=90=AF=E6=97=B6=E6=9F=A5=E4=B8=8D=E5=88=B0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=9C=A8=E6=8F=92=E4=BB=B6=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E5=AE=8C=E6=88=90=E5=90=8E=E5=BC=BA=E5=88=B6=E6=B8=85?= =?UTF-8?q?=E7=A9=BA=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 因为某些未知原因导致偶尔重启时查不到数据，在插件加载完成后强制清空缓存 --- .../conf/AutowireStaticSmartInitializingSingleton.java | 9 +++++++++ .../takin/web/biz/cache/AbstractAgentConfigCache.java | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java index 9af049ca15..be6f0fb529 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java @@ -1,11 +1,15 @@ package io.shulie.takin.web.app.conf; +import io.shulie.takin.web.biz.cache.AbstractAgentConfigCache; import io.shulie.takin.web.ext.util.WebPluginUtils; import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.AutowireCapableBeanFactory; import org.springframework.stereotype.Component; +import javax.annotation.Resource; +import java.util.List; + /** * @author by: hezhongqi * @Package io.shulie.takin.app.conf @@ -17,6 +21,8 @@ public class AutowireStaticSmartInitializingSingleton implements SmartInitializingSingleton { @Autowired private AutowireCapableBeanFactory beanFactory; + @Resource + private List cacheList; /** * 当所有的单例Bena初始化完成后，对static静态成员进行赋值 @@ -28,5 +34,8 @@ public class AutowireStaticSmartInitializingSingleton implements SmartInitializi public void afterSingletonsInstantiated() { // 因为是给static静态属性赋值，因此这里new一个实例做注入是可行的 beanFactory.autowireBean(new WebPluginUtils()); + // 因为某些未知原因导致偶尔重启时查不到数据，所以这里强制清空缓存，重新刷 + cacheList.forEach(AbstractAgentConfigCache::reset); + } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index e99d74ccb5..6d02d7b60e 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -48,7 +48,7 @@ public void evict(String namespace) { * 项目重启之后，缓存清空下 */ @PostConstruct - private void reset() { + public void reset() { String beClearKey = this.cacheName + "*"; if (!"*".equals(beClearKey)) { Set keys = redisTemplate.keys(beClearKey); From fd7dbfb94bb480eca1cde09149dd72e6bd1fbc76 Mon Sep 17 00:00:00 2001 From: nanfeng Date: Tue, 5 Jul 2022 10:09:30 +0800 Subject: [PATCH 07/84] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=97=B6=E5=A2=9E=E5=8A=A0=E5=BD=B1=E5=AD=90=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=B8=85=E9=99=A4=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/ClearDsDataConfig.java | 28 +++++++++++++++++++ ...owireStaticSmartInitializingSingleton.java | 9 ------ .../biz/cache/AbstractAgentConfigCache.java | 3 +- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java new file mode 100644 index 0000000000..6f8de4419b --- /dev/null +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java @@ -0,0 +1,28 @@ +package io.shulie.takin.web.app; + +import io.shulie.takin.web.biz.cache.AbstractAgentConfigCache; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @Author: 南风 + * @Date: 2022/7/5 9:36 上午 + */ +@Component +@Slf4j +public class ClearDsDataConfig implements ApplicationRunner { + + @Autowired + private List cacheList; + + @Override + public void run(ApplicationArguments args) throws Exception { + log.info("spring启动成功后重新清理缓存"); + cacheList.forEach(AbstractAgentConfigCache::reset); + } +} diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java index be6f0fb529..9af049ca15 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/AutowireStaticSmartInitializingSingleton.java @@ -1,15 +1,11 @@ package io.shulie.takin.web.app.conf; -import io.shulie.takin.web.biz.cache.AbstractAgentConfigCache; import io.shulie.takin.web.ext.util.WebPluginUtils; import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.AutowireCapableBeanFactory; import org.springframework.stereotype.Component; -import javax.annotation.Resource; -import java.util.List; - /** * @author by: hezhongqi * @Package io.shulie.takin.app.conf @@ -21,8 +17,6 @@ public class AutowireStaticSmartInitializingSingleton implements SmartInitializingSingleton { @Autowired private AutowireCapableBeanFactory beanFactory; - @Resource - private List cacheList; /** * 当所有的单例Bena初始化完成后，对static静态成员进行赋值 @@ -34,8 +28,5 @@ public class AutowireStaticSmartInitializingSingleton implements SmartInitializi public void afterSingletonsInstantiated() { // 因为是给static静态属性赋值，因此这里new一个实例做注入是可行的 beanFactory.autowireBean(new WebPluginUtils()); - // 因为某些未知原因导致偶尔重启时查不到数据，所以这里强制清空缓存，重新刷 - cacheList.forEach(AbstractAgentConfigCache::reset); - } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index 6d02d7b60e..03320be449 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -1,6 +1,7 @@ package io.shulie.takin.web.biz.cache; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; @@ -31,7 +32,7 @@ public T get(String namespace) { T result = (T)redisTemplate.opsForValue().get(getCacheKey(namespace)); if (result == null) { result = queryValue(namespace); - redisTemplate.opsForValue().set(getCacheKey(namespace), result); + redisTemplate.opsForValue().set(getCacheKey(namespace), result,5, TimeUnit.MINUTES); } return result; } From aac884cba308e47465c3098343cb5af8c4c880ed Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 11:32:47 +0800 Subject: [PATCH 08/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/app/conf/ElasticJobAutoConfig.java | 27 ++++ .../takin/web/app/conf/EsJobTaskListener.java | 136 ++++++++++++++++++ takin-web-common/pom.xml | 12 ++ .../shulie/takin/web/common/util/JVMUtil.java | 81 +++++++++++ 4 files changed, 256 insertions(+) create mode 100644 takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java create mode 100644 takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java index 02d683aa3e..af14db006e 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java @@ -2,6 +2,13 @@ import com.dangdang.ddframe.job.event.JobEventConfiguration; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobOperateAPI; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobStatisticsAPI; +import com.dangdang.ddframe.job.lite.lifecycle.internal.operate.JobOperateAPIImpl; +import com.dangdang.ddframe.job.lite.lifecycle.internal.statistics.JobStatisticsAPIImpl; +import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter; +import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration; +import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter; import org.springframework.context.annotation.Bean; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; @@ -45,4 +52,24 @@ public SpringJobSchedulerFactory springJobSchedulerFactory(ElasticJobProperties public JobConfParser jobConfParser(SpringJobSchedulerFactory springJobSchedulerFactory) { return new JobConfParser(springJobSchedulerFactory); } + + // 初始化elasticJob页面操作类 + @Bean + public CoordinatorRegistryCenter zkCenter() { + ZookeeperConfiguration configuration = new + ZookeeperConfiguration(zkAddress, "takin-web-job-" + env); + CoordinatorRegistryCenter center = new ZookeeperRegistryCenter(configuration); + center.init(); + return center; + } + + @Bean + public JobStatisticsAPI jobStatisticsAPI(CoordinatorRegistryCenter zkCenter) { + return new JobStatisticsAPIImpl(zkCenter); + } + + @Bean + public JobOperateAPI jobOperateAPI(CoordinatorRegistryCenter zkCenter) { + return new JobOperateAPIImpl(zkCenter); + } } diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java new file mode 100644 index 0000000000..60947e7e5e --- /dev/null +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -0,0 +1,136 @@ +package io.shulie.takin.web.app.conf; + +import com.dangdang.ddframe.job.lite.lifecycle.api.JobOperateAPI; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobStatisticsAPI; +import com.dangdang.ddframe.job.lite.lifecycle.domain.JobBriefInfo; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.shulie.takin.web.biz.service.DistributedLock; +import io.shulie.takin.web.common.util.JVMUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.File; +import java.io.FileOutputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.*; + +/** + * elasticjob 任务监听 + * + * @Date 7月5号 + */ +@Slf4j +@Component +public class EsJobTaskListener implements ApplicationListener { + // 需要监听的任务信息,按逗号分割 + @Value("${takin.listener.jobtask:CalcApplicationSummaryJob,CalcTpsTargetJob,FinishReportJob,SyncMachineDataJob}") + private String jobTask; + + // 是否打印jstack,如果状态不对的时候 + @Value("${takin.web.jstack.enable:true}") + private boolean jstackEnable; + + private static volatile long lastPrintTime = 0; + + private static final long TWO_MINUTES_MILLS = 2 * 60 * 1000; + + private static Semaphore guard = new Semaphore(1); + + private static final String USER_HOME = System.getProperty("user.home"); + + private static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd_HH:mm:ss"; + + @Resource + private JobOperateAPI jobOperateAPI; + + @Resource + private JobStatisticsAPI jobStatisticsAPI; + + @Resource + private DistributedLock distributedLock; + + /** + * @param event 事件源 + */ + @Override + public void onApplicationEvent(ApplicationStartedEvent event) { + try { + if (StringUtils.isNotBlank(jobTask)) { + ScheduledExecutorService service = + Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("JobTaskListener-%d").build()); + String esJobLock = "JobTaskListener_onApplicationEvent"; + // 检查job任务状态 + Runnable threadTask = new Runnable() { + @Override + public void run() { + try { + if (!distributedLock.tryLock(esJobLock, 1L, 1L, TimeUnit.MINUTES)) { + return; + } + // 获取任务状态 + String[] jobTaskArr = jobTask.split(","); + for (int i = 0; i < jobTaskArr.length; i++) { + // 获取任务状态 + JobBriefInfo jobBriefInfo = jobStatisticsAPI.getJobBriefInfo(jobTaskArr[i]); + // 任务状态非正常 + if (jobBriefInfo.getStatus() != JobBriefInfo.JobStatus.OK) { + log.error("当前任务状态不正常," + jobTaskArr[i]); + // 打印下jstack,看下当前线程在干啥 + if (jstackEnable) { + dumpJStack(); + } + // 点下触发，让他重试下 + jobOperateAPI.trigger(com.google.common.base.Optional.of(jobTaskArr[i]), null); + } + } + } catch (Throwable e) { + log.error("任务处理失败,{}" + ExceptionUtils.getStackTrace(e)); + } finally { + distributedLock.unLock(esJobLock); + } + } + }; + // 提交任务,10分钟以后，每隔30秒执行一次 + service.scheduleAtFixedRate(threadTask, 10 * 60, 10, TimeUnit.SECONDS); + } + } catch (Throwable e) { + log.error("JobTaskListener is fail,{}" + ExceptionUtils.getStackTrace(e)); + } + } + + private void dumpJStack() { + long now = System.currentTimeMillis(); + //dump every 2 minutes + if (now - lastPrintTime < TWO_MINUTES_MILLS) { + return; + } + if (!guard.tryAcquire()) { + return; + } + ExecutorService pool = Executors.newSingleThreadExecutor(); + pool.execute(() -> { + String dumpPath = USER_HOME; + SimpleDateFormat sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT); + String dateStr = sdf.format(new Date()); + //try-with-resources + try (FileOutputStream jStackStream = new FileOutputStream( + new File(dumpPath, "Takin_web_JStack.log" + "." + dateStr))) { + JVMUtil.jstack(jStackStream); + } catch (Throwable t) { + log.error("dump jStack error", t); + } finally { + guard.release(); + } + lastPrintTime = System.currentTimeMillis(); + }); + //must shutdown thread pool ,if not will lead to OOM + pool.shutdown(); + } +} diff --git a/takin-web-common/pom.xml b/takin-web-common/pom.xml index d2433f1701..fb3b78c6f8 100644 --- a/takin-web-common/pom.xml +++ b/takin-web-common/pom.xml @@ -137,6 +137,18 @@ module-pradar-core provided + + + com.dangdang + elastic-job-lite-lifecycle + + + org.eclipse.jetty.aggregate + jetty-all-server + + + 2.1.5 + diff --git a/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java b/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java new file mode 100644 index 0000000000..b67ec0830c --- /dev/null +++ b/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java @@ -0,0 +1,81 @@ +package io.shulie.takin.web.common.util; + +import java.io.OutputStream; +import java.lang.management.*; + +import static java.lang.Thread.State.*; + +/** + * @author zhouyuan + * @Date 2021年7月4号 + */ +public class JVMUtil { + public static void jstack(OutputStream stream) throws Exception { + ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); + for (ThreadInfo threadInfo : threadMxBean.dumpAllThreads(true, true)) { + stream.write(getThreadDumpString(threadInfo).getBytes()); + } + } + + private static String getThreadDumpString(ThreadInfo threadInfo) { + StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" + + " Id=" + threadInfo.getThreadId() + " " + + threadInfo.getThreadState()); + if (threadInfo.getLockName() != null) { + sb.append(" on " + threadInfo.getLockName()); + } + if (threadInfo.getLockOwnerName() != null) { + sb.append(" owned by \"" + threadInfo.getLockOwnerName() + + "\" Id=" + threadInfo.getLockOwnerId()); + } + if (threadInfo.isSuspended()) { + sb.append(" (suspended)"); + } + if (threadInfo.isInNative()) { + sb.append(" (in native)"); + } + sb.append('\n'); + int i = 0; + + StackTraceElement[] stackTrace = threadInfo.getStackTrace(); + MonitorInfo[] lockedMonitors = threadInfo.getLockedMonitors(); + for (; i < stackTrace.length && i < 32; i++) { + StackTraceElement ste = stackTrace[i]; + sb.append("\tat ").append(ste.toString()); + sb.append('\n'); + if (i == 0 && threadInfo.getLockInfo() != null) { + Thread.State ts = threadInfo.getThreadState(); + if (BLOCKED.equals(ts)) { + sb.append("\t- blocked on ").append(threadInfo.getLockInfo()); + sb.append('\n'); + } else if (WAITING.equals(ts) || TIMED_WAITING.equals(ts)) { + sb.append("\t- waiting on ").append(threadInfo.getLockInfo()); + sb.append('\n'); + } + } + + for (MonitorInfo mi : lockedMonitors) { + if (mi.getLockedStackDepth() == i) { + sb.append("\t- locked ").append(mi); + sb.append('\n'); + } + } + } + if (i < stackTrace.length) { + sb.append("\t..."); + sb.append('\n'); + } + + LockInfo[] locks = threadInfo.getLockedSynchronizers(); + if (locks.length > 0) { + sb.append("\n\tNumber of locked synchronizers = " + locks.length); + sb.append('\n'); + for (LockInfo li : locks) { + sb.append("\t- " + li); + sb.append('\n'); + } + } + sb.append('\n'); + return sb.toString(); + } +} \ No newline at end of file From 5ae4931e26ec3b75b99f16041ffbf4bd5a513a28 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 11:55:06 +0800 Subject: [PATCH 09/84] =?UTF-8?q?esjob=E8=B0=83=E6=95=B4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java | 2 +- .../java/io/shulie/takin/web/app/conf/EsJobTaskListener.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java index af14db006e..57b9ebe3ea 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java @@ -57,7 +57,7 @@ public JobConfParser jobConfParser(SpringJobSchedulerFactory springJobSchedulerF @Bean public CoordinatorRegistryCenter zkCenter() { ZookeeperConfiguration configuration = new - ZookeeperConfiguration(zkAddress, "takin-web-job-" + env); + ZookeeperConfiguration("192.168.1.185:2181", "takin-web-job-" + env); CoordinatorRegistryCenter center = new ZookeeperRegistryCenter(configuration); center.init(); return center; diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index 60947e7e5e..0e339abba1 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -30,7 +30,7 @@ @Component public class EsJobTaskListener implements ApplicationListener { // 需要监听的任务信息,按逗号分割 - @Value("${takin.listener.jobtask:CalcApplicationSummaryJob,CalcTpsTargetJob,FinishReportJob,SyncMachineDataJob}") + @Value("${takin.listener.jobtask:MavenNewVersionPullJob,CalcApplicationSummaryJob,CalcTpsTargetJob,FinishReportJob,SyncMachineDataJob}") private String jobTask; // 是否打印jstack,如果状态不对的时候 From 3fed79745605401f1961003e2a7ad5fe7ff0f130 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 12:34:19 +0800 Subject: [PATCH 10/84] =?UTF-8?q?esjob=E8=B0=83=E6=95=B4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java index 57b9ebe3ea..5eb7604bfe 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java @@ -57,7 +57,7 @@ public JobConfParser jobConfParser(SpringJobSchedulerFactory springJobSchedulerF @Bean public CoordinatorRegistryCenter zkCenter() { ZookeeperConfiguration configuration = new - ZookeeperConfiguration("192.168.1.185:2181", "takin-web-job-" + env); + ZookeeperConfiguration("192.168.1.185:2181", "takin-web-job-" + "_dev_28"); CoordinatorRegistryCenter center = new ZookeeperRegistryCenter(configuration); center.init(); return center; From 63abe713c16157c6e5d5e075aaa65afeeaf83749 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 13:23:13 +0800 Subject: [PATCH 11/84] =?UTF-8?q?esjob=E8=B0=83=E6=95=B4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shulie/takin/web/app/conf/EsJobTaskListener.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index 0e339abba1..95cc24b5b0 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -62,6 +62,7 @@ public class EsJobTaskListener implements ApplicationListener Date: Tue, 5 Jul 2022 14:16:25 +0800 Subject: [PATCH 12/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/conf/ElasticJobAutoConfig.java | 2 +- .../shulie/takin/web/app/conf/EsJobTaskListener.java | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java index 5eb7604bfe..af14db006e 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java @@ -57,7 +57,7 @@ public JobConfParser jobConfParser(SpringJobSchedulerFactory springJobSchedulerF @Bean public CoordinatorRegistryCenter zkCenter() { ZookeeperConfiguration configuration = new - ZookeeperConfiguration("192.168.1.185:2181", "takin-web-job-" + "_dev_28"); + ZookeeperConfiguration(zkAddress, "takin-web-job-" + env); CoordinatorRegistryCenter center = new ZookeeperRegistryCenter(configuration); center.init(); return center; diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index 95cc24b5b0..97106b98b3 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -30,7 +30,7 @@ @Component public class EsJobTaskListener implements ApplicationListener { // 需要监听的任务信息,按逗号分割 - @Value("${takin.listener.jobtask:MavenNewVersionPullJob,CalcApplicationSummaryJob,CalcTpsTargetJob,FinishReportJob,SyncMachineDataJob}") + @Value("${takin.listener.jobtask:calcApplicationSummaryJob,calcTpsTargetJob,finishReportJob,syncMachineDataJob}") private String jobTask; // 是否打印jstack,如果状态不对的时候 @@ -64,8 +64,7 @@ public void onApplicationEvent(ApplicationStartedEvent event) { try { log.info("EsJob 任务监听开启 "); if (StringUtils.isNotBlank(jobTask)) { - ScheduledExecutorService service = - Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("JobTaskListener-%d").build()); + ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("JobTaskListener-%d").build()); String esJobLock = "JobTaskListener_onApplicationEvent"; // 检查job任务状态 Runnable threadTask = new Runnable() { @@ -86,13 +85,12 @@ public void run() { } // 任务状态非正常 if (jobBriefInfo.getStatus() == JobBriefInfo.JobStatus.SHARDING_FLAG) { - log.error("当前任务状态不正常," + jobTaskArr[i]); // 打印下jstack,看下当前线程在干啥 if (jstackEnable) { dumpJStack(); } - // 点下触发，让他重试下 - log.info("触发当前任务,{}", jobTaskArr[i]); + // 点下触发,让他重试下 + log.info("当前任务状态为分片待调整,触发当前任务执行,{}", jobTaskArr[i]); jobOperateAPI.trigger(com.google.common.base.Optional.of(jobTaskArr[i]), null); } } From 4e485b5714b6d0cd796c7bcee43163e0d6092992 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 14:22:35 +0800 Subject: [PATCH 13/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/shulie/takin/web/app/conf/EsJobTaskListener.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index 97106b98b3..1e7b5dea75 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -24,17 +24,18 @@ /** * elasticjob 任务监听 * + * @author zhouyuan * @Date 7月5号 */ @Slf4j @Component public class EsJobTaskListener implements ApplicationListener { // 需要监听的任务信息,按逗号分割 - @Value("${takin.listener.jobtask:calcApplicationSummaryJob,calcTpsTargetJob,finishReportJob,syncMachineDataJob}") + @Value("${takin.listener.jobtask}") private String jobTask; // 是否打印jstack,如果状态不对的时候 - @Value("${takin.web.jstack.enable:true}") + @Value("${takin.web.jstack.enable}") private boolean jstackEnable; private static volatile long lastPrintTime = 0; @@ -102,7 +103,7 @@ public void run() { } }; // 提交任务,10分钟以后，每隔30秒执行一次 - service.scheduleAtFixedRate(threadTask, 10, 10, TimeUnit.SECONDS); + service.scheduleAtFixedRate(threadTask, 10 * 60, 30, TimeUnit.SECONDS); } } catch (Throwable e) { log.error("JobTaskListener is fail,{}" + ExceptionUtils.getStackTrace(e)); From 64a89859347a3169ee663b2670e254ed3d1c3a1c Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 14:42:51 +0800 Subject: [PATCH 14/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/conf/EsJobTaskListener.java | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index 1e7b5dea75..f2e277324a 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -119,23 +119,17 @@ private void dumpJStack() { if (!guard.tryAcquire()) { return; } - ExecutorService pool = Executors.newSingleThreadExecutor(); - pool.execute(() -> { - String dumpPath = USER_HOME; - SimpleDateFormat sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT); - String dateStr = sdf.format(new Date()); - //try-with-resources - try (FileOutputStream jStackStream = new FileOutputStream( - new File(dumpPath, "Takin_web_JStack.log" + "." + dateStr))) { - JVMUtil.jstack(jStackStream); - } catch (Throwable t) { - log.error("dump jStack error", t); - } finally { - guard.release(); - } - lastPrintTime = System.currentTimeMillis(); - }); - //must shutdown thread pool ,if not will lead to OOM - pool.shutdown(); + String dumpPath = USER_HOME; + SimpleDateFormat sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT); + String dateStr = sdf.format(new Date()); + try (FileOutputStream jStackStream = new FileOutputStream( + new File(dumpPath, "Takin_web_JStack.log" + "." + dateStr))) { + JVMUtil.jstack(jStackStream); + } catch (Throwable t) { + log.error("dump jStack error", t); + } finally { + guard.release(); + } + lastPrintTime = System.currentTimeMillis(); } } From 6516fc3f6f52d0ba5d37ee43b3ad6bc1876719ed Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 14:54:11 +0800 Subject: [PATCH 15/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/shulie/takin/web/app/conf/EsJobTaskListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index f2e277324a..c46ae5f32a 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -35,7 +35,7 @@ public class EsJobTaskListener implements ApplicationListener Date: Tue, 5 Jul 2022 16:22:42 +0800 Subject: [PATCH 16/84] bug fix:app --- .../service/impl/ApplicationServiceImpl.java | 28 +++++++++++++------ .../data/dao/application/ApplicationDAO.java | 2 +- .../dao/application/ApplicationDAOImpl.java | 2 +- .../mapper/mysql/ApplicationMntMapper.java | 2 +- .../mappers/ApplicationMntMapper.xml | 5 ++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 5a497878e5..28faddedb2 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -127,6 +127,7 @@ import java.util.*; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -756,21 +757,29 @@ public synchronized void syncApplicationAccessStatus() { private void syncApplicationAccessStatus(List applicationList,Set errorApplicationIdSet) { if (CollectionUtils.isNotEmpty(applicationList)) { applicationList.forEach(app -> { - Map result = applicationDAO.getStatus(app.getApplicationName()); - long n = (long) result.get("n"); - if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { - String e = (String) result.get("e"); + List list = applicationDAO.getStatus(app.getApplicationName()); + AtomicReference a = null; + AtomicReference result = null; + list.forEach(l->{ + String s = (String)l.get("s"); + if (org.apache.commons.lang3.StringUtils.equals(s,"3")) { + a.set((String) l.get("a")); + } else if (org.apache.commons.lang3.StringUtils.equals(s,"4")) { + result.set(l); + } + }); + if (null != result && (((long) result.get().get("n")) != 0 || (errorApplicationIdSet.contains(app.getApplicationId())))) { + String e = (String) result.get().get("e"); if (StringUtils.isBlank(e)) { - String a = (String)result.get("a"); e = "探针接入异常"; - if (StringUtils.isNotEmpty(a)) { - e += "，agentId为"+a; + if (null != a && StringUtils.isNotEmpty(a.get())) { + e += "，agentId为"+a.get(); } } applicationDAO.updateStatus(app.getApplicationId(), e); NodeUploadDataDTO param = new NodeUploadDataDTO(); param.setApplicationName(app.getApplicationName()); - param.setAgentId((String) result.get("a")); + param.setAgentId((String) result.get().get("a")); param.setNodeKey(UUID.randomUUID().toString().replace("_", "")); param.setExceptionTime(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())); HashMap map = new HashMap(1); @@ -782,7 +791,8 @@ private void syncApplicationAccessStatus(List application param.setSwitchErrorMap(map); uploadAccessStatus(param); } else { - applicationDAO.updateStatus(app.getApplicationId());} + applicationDAO.updateStatus(app.getApplicationId()); + } }); } } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java index 891ebf8d30..b222ba5458 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java @@ -428,7 +428,7 @@ public interface ApplicationDAO { PagingList pageByApplicationNamesAndUserId(Collection applicationNames, PageBaseDTO pageBaseDTO); - Map getStatus(String name); + List getStatus(String name); void updateStatus(Long applicationId, String e); diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java index 1f5c8e49fc..a60106911c 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java @@ -720,7 +720,7 @@ public PagingList pageByApplicationNamesAndUserId(Collect } @Override - public Map getStatus(String name) { + public List getStatus(String name) { return applicationMntMapper.getStatus(name); } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java index 5ceecda850..2dbdcfc353 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java @@ -232,5 +232,5 @@ IPage selectApplicationPageByParam( IPage selectApplicationListByUpgrade( @Param("page") IPage page, @Param("param") QueryApplicationByUpgradeParam param); - Map getStatus(@Param("name") String name); + List getStatus(@Param("name") String name); } diff --git a/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml b/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml index 518ce896c0..7821a065fa 100644 --- a/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml +++ b/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml @@ -439,7 +439,8 @@ ORDER BY a.id DESC - + SELECT agent_error_info AS e,COUNT(*) AS n,agent_id AS a,status AS s FROM t_agent_report WHERE application_name = #{name} GROUP BY `status` + From b55afd8f2ce2d38d92e500b2dc5bfcebd20330ac Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 5 Jul 2022 17:10:58 +0800 Subject: [PATCH 17/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/conf/EsJobTaskListener.java | 10 ++++-- .../biz/job/CalcApplicationSummaryJob.java | 30 +++++++++++----- .../takin/web/biz/job/CalcTpsTargetJob.java | 26 ++++++++++---- .../takin/web/biz/job/FinishReportJob.java | 34 +++++++++++++------ .../takin/web/biz/job/SyncMachineDataJob.java | 30 +++++++++++----- 5 files changed, 92 insertions(+), 38 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java index c46ae5f32a..5c553f13c3 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -34,6 +34,12 @@ public class EsJobTaskListener implements ApplicationListener taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } + if (taskDtoList == null) { + break; + } if (openVersion) { for (SceneTaskDto taskDto : taskDtoList) { Long reportId = taskDto.getReportId(); @@ -64,8 +76,8 @@ public void execute(ShardingContext shardingContext) { reportTaskService.calcApplicationSummary(reportId); } catch (Throwable e) { log.error( - "execute CalcApplicationSummaryJob occured error. reportId= {},errorMsg={}", - reportId, e.getMessage(), e); + "execute CalcApplicationSummaryJob occured error. reportId= {},errorMsg={}", + reportId, e.getMessage(), e); } finally { runningTasks.remove(reportId); } @@ -74,7 +86,7 @@ public void execute(ShardingContext shardingContext) { } } } else { - this.runTask(taskDtoList,shardingContext); + this.runTask(taskDtoList, shardingContext); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java index f02e062d8c..4a833d6f1b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java @@ -14,6 +14,7 @@ import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -24,11 +25,11 @@ */ @Component @ElasticSchedulerJob(jobName = "calcTpsTargetJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - //shardingItemParameters = "0=0,1=1,2=2", - isSharding = true, - cron = "*/10 * * * * ?", - description = "获取tps指标图") + // 分片序列号和参数用等号分隔 不需要参数可以不加 + //shardingItemParameters = "0=0,1=1,2=2", + isSharding = true, + cron = "*/10 * * * * ?", + description = "获取tps指标图") @Slf4j public class CalcTpsTargetJob extends AbstractSceneTask implements SimpleJob { @@ -44,11 +45,22 @@ public class CalcTpsTargetJob extends AbstractSceneTask implements SimpleJob { @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.CalcTpsTargetJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); while (true) { List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } + if (taskDtoList == null) { + break; + } if (openVersion) { for (SceneTaskDto taskDto : taskDtoList) { Long reportId = taskDto.getReportId(); @@ -70,7 +82,7 @@ public void execute(ShardingContext shardingContext) { } } } else { - this.runTask(taskDtoList,shardingContext); + this.runTask(taskDtoList, shardingContext); } } log.debug("calcTpsTargetJob 执行时间:{}", System.currentTimeMillis() - start); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java index 141e75bcea..b1d4678676 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java @@ -14,6 +14,7 @@ import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -24,11 +25,11 @@ */ @Component @ElasticSchedulerJob(jobName = "finishReportJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - //shardingItemParameters = "0=0,1=1,2=2", - isSharding = true, - cron = "*/10 * * * * ?", - description = "压测报告状态，汇总报告") + // 分片序列号和参数用等号分隔 不需要参数可以不加 + //shardingItemParameters = "0=0,1=1,2=2", + isSharding = true, + cron = "*/10 * * * * ?", + description = "压测报告状态，汇总报告") @Slf4j public class FinishReportJob extends AbstractSceneTask implements SimpleJob { @Autowired @@ -43,13 +44,24 @@ public class FinishReportJob extends AbstractSceneTask implements SimpleJob { @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.FinishReportJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); //任务开始 - while (true){ + while (true) { List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } - if(openVersion) { + if (taskDtoList == null) { + break; + } + if (openVersion) { for (SceneTaskDto taskDto : taskDtoList) { Long reportId = taskDto.getReportId(); // 私有化 + 开源 根据 报告id进行分片 @@ -59,7 +71,7 @@ public void execute(ShardingContext shardingContext) { if (task == null) { reportThreadPool.execute(() -> { try { - reportTaskService.finishReport(reportId,taskDto); + reportTaskService.finishReport(reportId, taskDto); } catch (Throwable e) { log.error("execute FinishReportJob occured error. reportId={}", reportId, e); } finally { @@ -70,7 +82,7 @@ public void execute(ShardingContext shardingContext) { } } this.cleanUnAvailableTasks(taskDtoList); - }else { + } else { final List taskAlreadyRun = this.runTask(taskDtoList, shardingContext); this.cleanUnAvailableTasks(taskAlreadyRun); } @@ -84,7 +96,7 @@ public void execute(ShardingContext shardingContext) { } @Override - protected void runTaskInTenantIfNecessary( SceneTaskDto tenantTask, Long reportId) { + protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 reportThreadPool.execute(() -> { try { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java index 9f4fe04298..397bff83ae 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java @@ -14,6 +14,7 @@ import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -24,11 +25,11 @@ */ @Component @ElasticSchedulerJob(jobName = "syncMachineDataJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - isSharding = true, - //shardingItemParameters = "0=0,1=1,2=2", - cron = "*/10 * * * * ?", - description = "同步应用基础信息") + // 分片序列号和参数用等号分隔 不需要参数可以不加 + isSharding = true, + //shardingItemParameters = "0=0,1=1,2=2", + cron = "*/10 * * * * ?", + description = "同步应用基础信息") @Slf4j public class SyncMachineDataJob extends AbstractSceneTask implements SimpleJob { @@ -45,13 +46,24 @@ public class SyncMachineDataJob extends AbstractSceneTask implements SimpleJob { @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.SyncMachineDataJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); while (true) { List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } + if (taskDtoList == null) { + break; + } - if (openVersion){ + if (openVersion) { for (SceneTaskDto taskDto : taskDtoList) { Long reportId = taskDto.getReportId(); if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { @@ -70,8 +82,8 @@ public void execute(ShardingContext shardingContext) { } } - }else { - this.runTask(taskDtoList,shardingContext); + } else { + this.runTask(taskDtoList, shardingContext); } } From cd8dc04bbf8a2a3d57bd66c0411867fb816672ba Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 17:29:01 +0800 Subject: [PATCH 18/84] =?UTF-8?q?bug=20fix:=E5=85=A5=E5=8F=A3=E5=8C=B9?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 28faddedb2..306a2c36b7 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -761,10 +761,10 @@ private void syncApplicationAccessStatus(List application AtomicReference a = null; AtomicReference result = null; list.forEach(l->{ - String s = (String)l.get("s"); - if (org.apache.commons.lang3.StringUtils.equals(s,"3")) { + int s = Integer.parseInt(l.get("s").toString()); + if (s == 3) { a.set((String) l.get("a")); - } else if (org.apache.commons.lang3.StringUtils.equals(s,"4")) { + } else if (s == 4) { result.set(l); } }); From 9c5d3c655cca748e3f28087fd9e93ab10e7c6d35 Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 17:52:56 +0800 Subject: [PATCH 19/84] bug fix:app --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 306a2c36b7..41d2805515 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -759,7 +759,7 @@ private void syncApplicationAccessStatus(List application applicationList.forEach(app -> { List list = applicationDAO.getStatus(app.getApplicationName()); AtomicReference a = null; - AtomicReference result = null; + AtomicReference result = new AtomicReference<>(); list.forEach(l->{ int s = Integer.parseInt(l.get("s").toString()); if (s == 3) { @@ -768,7 +768,7 @@ private void syncApplicationAccessStatus(List application result.set(l); } }); - if (null != result && (((long) result.get().get("n")) != 0 || (errorApplicationIdSet.contains(app.getApplicationId())))) { + if (null != result.get() && (((long) result.get().get("n")) != 0 || (errorApplicationIdSet.contains(app.getApplicationId())))) { String e = (String) result.get().get("e"); if (StringUtils.isBlank(e)) { e = "探针接入异常"; From 9baa8cee989bd02ad1837609cf8237b34f97587f Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 17:55:33 +0800 Subject: [PATCH 20/84] bug fix:app --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 41d2805515..98f08cd735 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -768,7 +768,7 @@ private void syncApplicationAccessStatus(List application result.set(l); } }); - if (null != result.get() && (((long) result.get().get("n")) != 0 || (errorApplicationIdSet.contains(app.getApplicationId())))) { + if ((null != result.get() && ((long) result.get().get("n")) != 0) || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = (String) result.get().get("e"); if (StringUtils.isBlank(e)) { e = "探针接入异常"; From 72461dde0d772472898fe7a88f2e6ae52def5897 Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 18:14:24 +0800 Subject: [PATCH 21/84] =?UTF-8?q?bug=20fix:=E5=85=A5=E5=8F=A3=E5=8C=B9?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 98f08cd735..a2c6d4aea8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -769,12 +769,14 @@ private void syncApplicationAccessStatus(List application } }); if ((null != result.get() && ((long) result.get().get("n")) != 0) || (errorApplicationIdSet.contains(app.getApplicationId()))) { - String e = (String) result.get().get("e"); - if (StringUtils.isBlank(e)) { + String e = ""; + if (null == result || StringUtils.isBlank((String) result.get().get("e"))) { e = "探针接入异常"; if (null != a && StringUtils.isNotEmpty(a.get())) { e += "，agentId为"+a.get(); } + } else if (null != result && StringUtils.isNotBlank((String) result.get().get("e"))) { + e = (String) result.get().get("e"); } applicationDAO.updateStatus(app.getApplicationId(), e); NodeUploadDataDTO param = new NodeUploadDataDTO(); From 8bb77de559c3bdf762ac1508464ef626a01adfb3 Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 18:25:48 +0800 Subject: [PATCH 22/84] bug fix:app --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index a2c6d4aea8..1172d2f913 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -770,12 +770,12 @@ private void syncApplicationAccessStatus(List application }); if ((null != result.get() && ((long) result.get().get("n")) != 0) || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = ""; - if (null == result || StringUtils.isBlank((String) result.get().get("e"))) { + if (null == result.get() || StringUtils.isBlank((String) result.get().get("e"))) { e = "探针接入异常"; if (null != a && StringUtils.isNotEmpty(a.get())) { e += "，agentId为"+a.get(); } - } else if (null != result && StringUtils.isNotBlank((String) result.get().get("e"))) { + } else if (null != result.get() && StringUtils.isNotBlank((String) result.get().get("e"))) { e = (String) result.get().get("e"); } applicationDAO.updateStatus(app.getApplicationId(), e); From 842183a9883d174205b95bb2908785cdf7c02958 Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 18:49:16 +0800 Subject: [PATCH 23/84] bug fix:app --- .../takin/web/biz/service/impl/ApplicationServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 1172d2f913..bbeb0e096b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -758,7 +758,7 @@ private void syncApplicationAccessStatus(List application if (CollectionUtils.isNotEmpty(applicationList)) { applicationList.forEach(app -> { List list = applicationDAO.getStatus(app.getApplicationName()); - AtomicReference a = null; + AtomicReference a = new AtomicReference<>(); AtomicReference result = new AtomicReference<>(); list.forEach(l->{ int s = Integer.parseInt(l.get("s").toString()); @@ -781,7 +781,7 @@ private void syncApplicationAccessStatus(List application applicationDAO.updateStatus(app.getApplicationId(), e); NodeUploadDataDTO param = new NodeUploadDataDTO(); param.setApplicationName(app.getApplicationName()); - param.setAgentId((String) result.get().get("a")); + param.setAgentId(a.get()); param.setNodeKey(UUID.randomUUID().toString().replace("_", "")); param.setExceptionTime(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())); HashMap map = new HashMap(1); From e59088e048306019fb95e576b3c887c027579eab Mon Sep 17 00:00:00 2001 From: zhangjunjie Date: Tue, 5 Jul 2022 19:00:58 +0800 Subject: [PATCH 24/84] bug fix:reset --- .../service/impl/ApplicationServiceImpl.java | 27 +++++++------------ .../data/dao/application/ApplicationDAO.java | 2 +- .../dao/application/ApplicationDAOImpl.java | 2 +- .../mapper/mysql/ApplicationMntMapper.java | 2 +- .../mappers/ApplicationMntMapper.xml | 5 ++-- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 28faddedb2..823c8bf4e1 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -757,29 +757,21 @@ public synchronized void syncApplicationAccessStatus() { private void syncApplicationAccessStatus(List applicationList,Set errorApplicationIdSet) { if (CollectionUtils.isNotEmpty(applicationList)) { applicationList.forEach(app -> { - List list = applicationDAO.getStatus(app.getApplicationName()); - AtomicReference a = null; - AtomicReference result = null; - list.forEach(l->{ - String s = (String)l.get("s"); - if (org.apache.commons.lang3.StringUtils.equals(s,"3")) { - a.set((String) l.get("a")); - } else if (org.apache.commons.lang3.StringUtils.equals(s,"4")) { - result.set(l); - } - }); - if (null != result && (((long) result.get().get("n")) != 0 || (errorApplicationIdSet.contains(app.getApplicationId())))) { - String e = (String) result.get().get("e"); + Map result = applicationDAO.getStatus(app.getApplicationName()); + long n = (long) result.get("n"); + if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { + String e = (String) result.get("e"); if (StringUtils.isBlank(e)) { + String a = (String)result.get("a"); e = "探针接入异常"; - if (null != a && StringUtils.isNotEmpty(a.get())) { - e += "，agentId为"+a.get(); + if (StringUtils.isNotEmpty(a)) { + e += "，agentId为"+a; } } applicationDAO.updateStatus(app.getApplicationId(), e); NodeUploadDataDTO param = new NodeUploadDataDTO(); param.setApplicationName(app.getApplicationName()); - param.setAgentId((String) result.get().get("a")); + param.setAgentId((String) result.get("a")); param.setNodeKey(UUID.randomUUID().toString().replace("_", "")); param.setExceptionTime(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())); HashMap map = new HashMap(1); @@ -791,8 +783,7 @@ private void syncApplicationAccessStatus(List application param.setSwitchErrorMap(map); uploadAccessStatus(param); } else { - applicationDAO.updateStatus(app.getApplicationId()); - } + applicationDAO.updateStatus(app.getApplicationId());} }); } } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java index b222ba5458..891ebf8d30 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAO.java @@ -428,7 +428,7 @@ public interface ApplicationDAO { PagingList pageByApplicationNamesAndUserId(Collection applicationNames, PageBaseDTO pageBaseDTO); - List getStatus(String name); + Map getStatus(String name); void updateStatus(Long applicationId, String e); diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java index a60106911c..1f5c8e49fc 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationDAOImpl.java @@ -720,7 +720,7 @@ public PagingList pageByApplicationNamesAndUserId(Collect } @Override - public List getStatus(String name) { + public Map getStatus(String name) { return applicationMntMapper.getStatus(name); } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java index 2dbdcfc353..5ceecda850 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/mapper/mysql/ApplicationMntMapper.java @@ -232,5 +232,5 @@ IPage selectApplicationPageByParam( IPage selectApplicationListByUpgrade( @Param("page") IPage page, @Param("param") QueryApplicationByUpgradeParam param); - List getStatus(@Param("name") String name); + Map getStatus(@Param("name") String name); } diff --git a/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml b/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml index 7821a065fa..518ce896c0 100644 --- a/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml +++ b/takin-web-data/src/main/resources/mappers/ApplicationMntMapper.xml @@ -439,8 +439,7 @@ ORDER BY a.id DESC - + SELECT agent_error_info AS e,COUNT(*) AS n,agent_id AS a FROM t_agent_report WHERE application_name = #{name} AND `status` = 4 - From d582e3c58aa6abc9020381f3f6ad86af37cb04d1 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 11:09:03 +0800 Subject: [PATCH 25/84] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=A4=84=E7=90=86=E7=BA=BF=E7=A8=8B=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/biz/common/AbstractSceneTask.java | 40 +++++-- .../biz/job/CalcApplicationSummaryJob.java | 68 ++++------- .../takin/web/biz/job/CalcTpsTargetJob.java | 61 ++++------ .../takin/web/biz/job/FinishReportJob.java | 67 +++++------ .../takin/web/biz/job/SyncMachineDataJob.java | 62 ++++------ .../report/impl/ReportTaskServiceImpl.java | 35 +++--- .../web/biz/threadpool/ThreadPoolUtil.java | 111 ++++++++++++++++++ 7 files changed, 264 insertions(+), 180 deletions(-) create mode 100644 takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java index 98f7b768ab..30ba938ca5 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java @@ -1,11 +1,8 @@ package io.shulie.takin.web.biz.common; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -63,7 +60,7 @@ protected int getAllowedTenantThreadMax() { return allowedTenantThreadMax; } allowedTenantThreadMax = ConfigServerHelper.getIntegerValueByKey( - ConfigServerKeyEnum.PER_TENANT_ALLOW_TASK_THREADS_MAX); + ConfigServerKeyEnum.PER_TENANT_ALLOW_TASK_THREADS_MAX); return allowedTenantThreadMax; } @@ -72,7 +69,7 @@ protected void cleanUnAvailableTasks(List taskDtoList) { if (CollectionUtils.isNotEmpty(taskDtoList)) { final LocalDateTime now = LocalDateTime.now(); taskDtoList.stream().filter(t -> t.getEndTime() != null && now.compareTo(t.getEndTime()) > 0).forEach( - t -> removeReportKey(t.getReportId())); + t -> removeReportKey(t.getReportId())); } } catch (Exception e) { log.error("清理过期任务时发生错误！", e); @@ -96,7 +93,7 @@ protected synchronized List runTask(List taskDtoList int allowedThreadMax = this.getAllowedTenantThreadMax(); //筛选出租户的任务 final Map> listMap = taskDtoList.stream().filter(t -> - t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() + t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() ).collect(Collectors.groupingBy(SceneTaskDto::getTenantId)); if (org.springframework.util.CollectionUtils.isEmpty(listMap)) { return taskAlreadyRun; @@ -111,7 +108,7 @@ protected synchronized List runTask(List taskDtoList * 取最值。当前租户的任务数和允许的最大线程数 */ AtomicInteger allowRunningThreads = new AtomicInteger( - Math.min(allowedThreadMax, tenantTasks.size())); + Math.min(allowedThreadMax, tenantTasks.size())); /** * 已经运行的任务数 @@ -128,7 +125,7 @@ protected synchronized List runTask(List taskDtoList * allow running threads calculated by capacity */ int permitsThreads = Math.min(allowedThreadMax - oldRunningThreads.get(), - allowRunningThreads.get()); + allowRunningThreads.get()); // add new threads to capacity oldRunningThreads.addAndGet(permitsThreads); // adjust allow current running threads @@ -144,4 +141,29 @@ protected synchronized List runTask(List taskDtoList return taskAlreadyRun; } + + /** + * @param taskDtoList + * @param shardingContext + */ + protected void runTask_ext(List taskDtoList, ShardingContext shardingContext) { + //筛选出租户的任务 + final Map> listMap = + taskDtoList.stream().filter(t -> t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() + ).collect(Collectors.groupingBy(SceneTaskDto::getTenantId)); + if (listMap.isEmpty()) { + return; + } + for (Entry> listEntry : listMap.entrySet()) { + final List tenantTasks = listEntry.getValue(); + if (CollectionUtils.isEmpty(tenantTasks)) { + continue; + } + for (int i = 0; i < tenantTasks.size(); i++) { + final SceneTaskDto task = tenantTasks.get(i); + this.runTaskInTenantIfNecessary(task, task.getReportId()); + } + } + } + } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java index 151c7d5b85..ced0f04c87 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java @@ -2,25 +2,21 @@ import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import com.dangdang.ddframe.job.api.ShardingContext; import com.dangdang.ddframe.job.api.simple.SimpleJob; import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; /** * @author 无涯 @@ -39,10 +35,6 @@ public class CalcApplicationSummaryJob extends AbstractSceneTask implements Simp @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportSummaryThreadPool") - private ThreadPoolExecutor reportThreadPool; - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @@ -59,49 +51,41 @@ public void execute(ShardingContext shardingContext) { public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { - break; - } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.calcApplicationSummary(reportId); - } catch (Throwable e) { - log.error( - "execute CalcApplicationSummaryJob occured error. reportId= {},errorMsg={}", - reportId, e.getMessage(), e); - } finally { - runningTasks.remove(reportId); - } - }); - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportSummaryThreadPool().execute(() -> { + try { + reportTaskService.calcApplicationSummary(reportId); + } catch (Throwable e) { + log.error( + "execute CalcApplicationSummaryJob occured error. reportId= {},errorMsg={}", + reportId, e.getMessage(), e); + } finally { + runningTasks.remove(reportId); + } + }); } } - } else { - this.runTask(taskDtoList, shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } - log.debug("calcApplicationSummaryJob 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getReportSummaryThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.calcApplicationSummary(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java index 4a833d6f1b..20d0dbb00f 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; @@ -11,12 +10,12 @@ import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** @@ -37,8 +36,7 @@ public class CalcTpsTargetJob extends AbstractSceneTask implements SimpleJob { private ReportTaskService reportTaskService; @Autowired - @Qualifier("reportTpsThreadPool") - private ThreadPoolExecutor reportThreadPool; + private ThreadPoolUtil threadPoolUtil; private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @@ -56,47 +54,40 @@ public void execute(ShardingContext shardingContext) { public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { - break; - } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.calcTpsTarget(reportId); - } catch (Throwable e) { - log.error("execute CalcTpsTargetJob occured error. reportId={}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - - }); - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current task is null "); + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportTpsThreadPool().execute(() -> { + try { + reportTaskService.calcTpsTarget(reportId); + } catch (Throwable e) { + log.error("execute CalcTpsTargetJob occured error. reportId={}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } } - } else { - this.runTask(taskDtoList, shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } log.debug("calcTpsTargetJob 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + threadPoolUtil.getReportTpsThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.calcTpsTarget(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java index b1d4678676..1c2a93e335 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; @@ -11,12 +10,12 @@ import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** @@ -35,10 +34,6 @@ public class FinishReportJob extends AbstractSceneTask implements SimpleJob { @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportFinishThreadPool") - private ThreadPoolExecutor reportThreadPool; - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @@ -56,49 +51,43 @@ public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); //任务开始 - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { - break; - } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 私有化 + 开源 根据 报告id进行分片 - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.finishReport(reportId, taskDto); - } catch (Throwable e) { - log.error("execute FinishReportJob occured error. reportId={}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - }); - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("TaskList is null"); + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 私有化 + 开源 根据 报告id进行分片 + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportFinishThreadPool().execute(() -> { + try { + reportTaskService.finishReport(reportId, taskDto); + } catch (Throwable e) { + log.error("execute FinishReportJob occured error. reportId={}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } } - this.cleanUnAvailableTasks(taskDtoList); - } else { - final List taskAlreadyRun = this.runTask(taskDtoList, shardingContext); - this.cleanUnAvailableTasks(taskAlreadyRun); } + this.cleanUnAvailableTasks(taskDtoList); + } else { + this.runTask_ext(taskDtoList, shardingContext); + this.cleanUnAvailableTasks(taskDtoList); } log.debug("finishReport 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getReportFinishThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.finishReport(reportId, tenantTask); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java index 397bff83ae..cc2b765ac8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; @@ -11,12 +10,12 @@ import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** @@ -36,11 +35,6 @@ public class SyncMachineDataJob extends AbstractSceneTask implements SimpleJob { @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportMachineThreadPool") - private ThreadPoolExecutor reportThreadPool; - - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @@ -57,48 +51,40 @@ public void execute(ShardingContext shardingContext) { public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { - break; - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current task is null "); + return; + } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.syncMachineData(reportId); - } catch (Throwable e) { - log.error("execute SyncMachineDataJob occured error. reportId= {}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - }); - } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getSyncMachinePool().execute(() -> { + try { + reportTaskService.syncMachineData(reportId); + } catch (Throwable e) { + log.error("execute SyncMachineDataJob occured error. reportId= {}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } - } - } else { - this.runTask(taskDtoList, shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } - log.debug("syncMachineData 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getSyncMachinePool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.syncMachineData(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java index 6f006d01df..5c9a3ae745 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java @@ -3,7 +3,6 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Date; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import com.alibaba.fastjson.JSON; @@ -13,6 +12,7 @@ import io.shulie.takin.cloud.sdk.model.request.report.UpdateReportConclusionReq; import io.shulie.takin.common.beans.response.ResponseResult; import io.shulie.takin.web.biz.constant.WebRedisKeyConstant; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.biz.pojo.output.report.ReportDetailOutput; import io.shulie.takin.web.biz.service.DistributedLock; import io.shulie.takin.web.biz.service.report.ReportService; @@ -41,7 +41,7 @@ * - 风险机器 * - 容量水位 * 4、更新报告状态为已完成 - * + *

* 配合压测实况：容量水位 * 1、压测中生成报告，执行 机器列表 tps汇总图 机器统计 * 2、压测中时，别忘记解锁 @@ -78,8 +78,7 @@ public class ReportTaskServiceImpl implements ReportTaskService { private SceneTaskApi sceneTaskApi; @Autowired - @Qualifier("collectDataThreadPool") - private ThreadPoolExecutor collectDataThreadPool; + private ThreadPoolUtil threadPoolUtil; @Autowired @Qualifier("redisTemplate") @@ -104,10 +103,10 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { } // 加锁 // 分布式锁 - String lockKey = JobRedisUtils.getRedisJobReport(WebPluginUtils.traceTenantId(), WebPluginUtils.traceEnvCode(),reportId); + String lockKey = JobRedisUtils.getRedisJobReport(WebPluginUtils.traceTenantId(), WebPluginUtils.traceEnvCode(), reportId); if (!distributedLock.checkLock(lockKey)) { // 收集数据 单独线程收集 - collectDataThreadPool.execute(collectData(reportId,commonExt,lockKey)); + threadPoolUtil.getCollectDataThreadPool().execute(collectData(reportId, commonExt, lockKey)); } // 压测结束才锁报告 Integer status = report.getTaskStatus(); @@ -116,12 +115,14 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { } ReportDetailDTO reportDetailDTO = reportDataCache.getReportDetailDTO(reportId); if (reportDetailDTO == null) { - log.error("未查到报告明细！reportId={}",reportId); + log.error("未查到报告明细！reportId={}", reportId); return false; } Date endTime = reportDetailDTO.getEndTime(); //更新任务的结束时间 - if (!this.updateTaskEndTime(reportId, commonExt, endTime)) { return false; } + if (!this.updateTaskEndTime(reportId, commonExt, endTime)) { + return false; + } // 解除 场景锁 redisClientUtils.delete(SceneTaskUtils.getSceneTaskKey(reportDetailDTO.getSceneId())); @@ -131,8 +132,8 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { redisClientUtils.del(WebRedisKeyConstant.REPORT_WARN_PREFIX + reportId); // 删除key String redisKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); redisClientUtils.del(redisKey); long startTime = System.currentTimeMillis(); Boolean lockResponse = reportService.lockReport(reportId); @@ -168,7 +169,7 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { //压测结束，生成压测报告异常，解锁报告 Boolean unLockReportResult = reportService.unLockReport(reportId); log.error("Unlock Report Success, reportId={} ,unLockReportResult= {}...", reportId, unLockReportResult, - e); + e); } finally { removeReportKey(reportId, commonExt); } @@ -215,10 +216,10 @@ private void removeReportKey(Long reportId, TenantCommonExt commonExt) { * @param reportId 报告 id * @return 可运行 */ - private synchronized Runnable collectData(Long reportId, TenantCommonExt commonExt,String lockKey) { + private synchronized Runnable collectData(Long reportId, TenantCommonExt commonExt, String lockKey) { return () -> { - boolean tryLock = distributedLock.tryLock(lockKey, 1L, 1L, TimeUnit.MINUTES); - if(!tryLock) { + boolean tryLock = distributedLock.tryLock(lockKey, 10L, 10L, TimeUnit.SECONDS); + if (!tryLock) { return; } WebPluginUtils.setTraceTenantContext(commonExt); @@ -253,7 +254,7 @@ public void syncMachineData(Long reportId) { long startTime = System.currentTimeMillis(); problemAnalysisService.syncMachineData(reportId); log.debug("reportId={} syncMachineData success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } @Override @@ -264,7 +265,7 @@ public void calcTpsTarget(Long reportId) { //then tps指标图 summaryService.calcTpsTarget(reportId); log.debug("reportId={} calcTpsTarget success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } @Override @@ -275,6 +276,6 @@ public void calcApplicationSummary(Long reportId) { //汇总应用 机器数 风险机器数 summaryService.calcApplicationSummary(reportId); log.debug("reportId={} calcApplicationSummary success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java new file mode 100644 index 0000000000..cf8aeadc7d --- /dev/null +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java @@ -0,0 +1,111 @@ +package io.shulie.takin.web.biz.threadpool; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author xingchen + * @description: TODO + * @date 2022/7/7 9:32 AM + */ +@Component +public class ThreadPoolUtil implements ApplicationContextAware { + private static ApplicationContext applicationContext; + + List syncMachineList = + Arrays.asList("reportMachineThreadPool", + "reportMachineThreadPool_one", + "reportMachineThreadPool_two", + "reportMachineThreadPool_three"); + + List reportTpsList = + Arrays.asList("reportTpsThreadPool", + "reportTpsThreadPool_one", + "reportTpsThreadPool_two", + "reportTpsThreadPool_three"); + + List reportSummaryList = + Arrays.asList("reportSummaryThreadPool", + "reportSummaryThreadPool_one", + "reportSummaryThreadPool_two", + "reportSummaryThreadPool_three"); + + List reportFinishList = + Arrays.asList("reportFinishThreadPool", + "reportFinishThreadPool_one", + "reportFinishThreadPool_two", + "reportFinishThreadPool_three"); + + List collectDataList = + Arrays.asList("collectDataThreadPool", + "collectDataThreadPool_one", + "collectDataThreadPool_two", + "collectDataThreadPool_three"); + + private static AtomicLong atomicMachine = new AtomicLong(0); + private static AtomicLong atomicTps = new AtomicLong(0); + private static AtomicLong atomicSummary = new AtomicLong(0); + + static List syncMachineDataJobThreadPools = new ArrayList<>(); + static List reportTpsThreadPools = new ArrayList<>(); + static List reportSummaryThreadPools = new ArrayList<>(); + static List reportFinishThreadPools = new ArrayList<>(); + static List collectDataThreadPools = new ArrayList<>(); + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @PostConstruct + public void init() { + syncMachineList.stream().forEach(pool -> { + syncMachineDataJobThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportTpsList.stream().forEach(pool -> { + reportTpsThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportSummaryList.stream().forEach(pool -> { + reportSummaryThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportFinishList.stream().forEach(pool -> { + reportFinishThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + collectDataList.stream().forEach(pool -> { + collectDataThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + } + + public static ThreadPoolExecutor getSyncMachinePool() { + return syncMachineDataJobThreadPools.get((int) Math.abs(atomicMachine.getAndIncrement() % syncMachineDataJobThreadPools.size())); + } + + public static ThreadPoolExecutor getReportTpsThreadPool() { + return reportTpsThreadPools.get((int) Math.abs(atomicTps.getAndIncrement() % reportTpsThreadPools.size())); + } + + public static ThreadPoolExecutor getReportSummaryThreadPool() { + return reportSummaryThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % reportSummaryThreadPools.size())); + } + + public static ThreadPoolExecutor getReportFinishThreadPool() { + return reportFinishThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % reportFinishThreadPools.size())); + } + + public static ThreadPoolExecutor getCollectDataThreadPool() { + return collectDataThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % collectDataThreadPools.size())); + } +} From 7f1b7abcff1b7bf5a3c95f18309132c0eae5149e Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 11:26:38 +0800 Subject: [PATCH 26/84] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=A4=84=E7=90=86=E7=BA=BF=E7=A8=8B=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/conf/ThreadPoolConfig.java | 151 +++++++++++++++--- 1 file changed, 125 insertions(+), 26 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java index d401c85bb2..e604e875bf 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java @@ -41,7 +41,7 @@ public class ThreadPoolConfig { public ThreadPoolExecutor jobThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-%d").build(); return new ThreadPoolExecutor(5, 10, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -53,7 +53,7 @@ public ThreadPoolExecutor jobThreadPool() { public ThreadPoolExecutor traceManageThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-trace-%d").build(); return new ThreadPoolExecutor(5, 10, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -65,7 +65,28 @@ public ThreadPoolExecutor traceManageThreadPool() { public ThreadPoolExecutor collectDataThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_one") + public ThreadPoolExecutor collectDataThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); + return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_two") + public ThreadPoolExecutor collectDataThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); + return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_three") + public ThreadPoolExecutor collectDataThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); + return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -77,7 +98,7 @@ public ThreadPoolExecutor collectDataThreadPool() { public ThreadPoolExecutor showdownVerifyThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-showdown-verify-%d").build(); return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -89,8 +110,26 @@ public ThreadPoolExecutor showdownVerifyThreadPool() { @Bean(name = "reportSummaryThreadPool") public ThreadPoolExecutor reportSummaryThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportSummaryThreadPool_one") + public ThreadPoolExecutor reportSummaryThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportSummaryThreadPool_two") + public ThreadPoolExecutor reportSummaryThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportSummaryThreadPool_three") + public ThreadPoolExecutor reportSummaryThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -102,8 +141,28 @@ public ThreadPoolExecutor reportSummaryThreadPool() { @Bean(name = "reportTpsThreadPool") public ThreadPoolExecutor reportTpsThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportTpsThreadPool_one") + public ThreadPoolExecutor reportTpsThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportTpsThreadPool_two") + public ThreadPoolExecutor reportTpsThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportTpsThreadPool_three") + public ThreadPoolExecutor reportTpsThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -116,7 +175,26 @@ public ThreadPoolExecutor reportTpsThreadPool() { public ThreadPoolExecutor reportFinishThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportFinishThreadPool_one") + public ThreadPoolExecutor reportFinishThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + @Bean(name = "reportFinishThreadPool_two") + public ThreadPoolExecutor reportFinishThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportFinishThreadPool_three") + public ThreadPoolExecutor reportFinishThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -128,8 +206,29 @@ public ThreadPoolExecutor reportFinishThreadPool() { @Bean(name = "reportMachineThreadPool") public ThreadPoolExecutor reportMachineThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportMachineThreadPool_one") + public ThreadPoolExecutor reportMachineThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportMachineThreadPool_two") + public ThreadPoolExecutor reportMachineThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportMachineThreadPool_three") + public ThreadPoolExecutor reportMachineThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -141,14 +240,14 @@ public ThreadPoolExecutor reportMachineThreadPool() { public ThreadPoolExecutor fastDebug() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("fast-debug-%d").build(); return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "modifyMonitorThreadPool") public ThreadPoolExecutor modifyMonitorExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("modify-monitor-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "schedulerPool") @@ -156,7 +255,7 @@ public TaskScheduler scheduler() { ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler(); taskScheduler.setPoolSize(5); ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("pradar-scheduler-thread-%d") - .daemon(true).build(); + .daemon(true).build(); taskScheduler.setThreadFactory(threadFactory); taskScheduler.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); return taskScheduler; @@ -166,29 +265,29 @@ public TaskScheduler scheduler() { public ThreadPoolExecutor runShellTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("script-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "loadDataThreadPool") public ThreadPoolExecutor loadDataTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("loaddata-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "agentDataThreadPool") public ThreadPoolExecutor agentDataTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agentdata-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "backgroundMonitorThreadPool") public ThreadPoolExecutor backgroundMonitorThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("background-monitor-thread-%d") - .build(); + .build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean("asyncExecuteScriptThreadPool") @@ -216,7 +315,7 @@ public Executor myAsync() { public ThreadPoolExecutor runOPSShellTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("ops-script-thread-%d").build(); return new ThreadPoolExecutor(1, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Primary @@ -258,7 +357,7 @@ public HttpMessageConverters restFileDownloadSupport() { public ThreadPoolExecutor queryAsyncThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("query-async-thread-%d").build(); return new ThreadPoolExecutor(1, 10, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Value("${poolConfig.e2e.coreSize: 20}") @@ -279,7 +378,7 @@ public ThreadPoolExecutor queryAsyncThreadPool() { public ThreadPoolExecutor e2eThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("e2e-job-%d").build(); return new ThreadPoolExecutor(e2eCoreSize, e2eMaxSize, 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(e2eQueueSize), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + new ArrayBlockingQueue<>(e2eQueueSize), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -291,7 +390,7 @@ public ThreadPoolExecutor e2eThreadPool() { public ThreadPoolExecutor agentAggregationThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agent-aggregation-%d").build(); return new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -304,7 +403,7 @@ public ThreadPoolExecutor middlewareResolverThreadPool() { final int coreSize = Runtime.getRuntime().availableProcessors(); ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("middleware-resolver-%d").build(); return new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -317,8 +416,8 @@ public ExecutorService agentHeartbeatThreadPool() { final int coreSize = Runtime.getRuntime().availableProcessors(); ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agent-heartbeat-%d").build(); return TtlExecutors.getTtlExecutorService( - new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.DiscardPolicy())); + new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), + nameThreadFactory, new ThreadPoolExecutor.DiscardPolicy())); } /** From db7f03d3a19dc4d7a0176e0d1587d082636c7e35 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 12:37:28 +0800 Subject: [PATCH 27/84] =?UTF-8?q?esjob-=E7=8A=B6=E6=80=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/conf/ThreadPoolConfig.java | 63 +++++++------------ .../takin/web/biz/job/SyncMachineDataJob.java | 4 ++ .../report/impl/ReportLocalServiceImpl.java | 12 +++- .../report/impl/ReportTaskServiceImpl.java | 27 +++++--- .../web/biz/threadpool/ThreadPoolUtil.java | 18 +++--- 5 files changed, 65 insertions(+), 59 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java index e604e875bf..8746f92df2 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java @@ -64,28 +64,28 @@ public ThreadPoolExecutor traceManageThreadPool() { @Bean(name = "collectDataThreadPool") public ThreadPoolExecutor collectDataThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); - return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "collectDataThreadPool_one") public ThreadPoolExecutor collectDataThreadPool_one() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); - return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-two-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "collectDataThreadPool_two") public ThreadPoolExecutor collectDataThreadPool_two() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); - return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-two-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "collectDataThreadPool_three") public ThreadPoolExecutor collectDataThreadPool_three() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); - return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-three-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @@ -113,21 +113,24 @@ public ThreadPoolExecutor reportSummaryThreadPool() { return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } + @Bean(name = "reportSummaryThreadPool_one") public ThreadPoolExecutor reportSummaryThreadPool_one() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_one-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } + @Bean(name = "reportSummaryThreadPool_two") public ThreadPoolExecutor reportSummaryThreadPool_two() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_two-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } + @Bean(name = "reportSummaryThreadPool_three") public ThreadPoolExecutor reportSummaryThreadPool_three() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_three-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @@ -147,20 +150,14 @@ public ThreadPoolExecutor reportTpsThreadPool() { @Bean(name = "reportTpsThreadPool_one") public ThreadPoolExecutor reportTpsThreadPool_one() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-one-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "reportTpsThreadPool_two") public ThreadPoolExecutor reportTpsThreadPool_two() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - } - @Bean(name = "reportTpsThreadPool_three") - public ThreadPoolExecutor reportTpsThreadPool_three() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-two-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @@ -174,26 +171,21 @@ public ThreadPoolExecutor reportTpsThreadPool_three() { @Bean(name = "reportFinishThreadPool") public ThreadPoolExecutor reportFinishThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } + @Bean(name = "reportFinishThreadPool_one") public ThreadPoolExecutor reportFinishThreadPool_one() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-one-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } + @Bean(name = "reportFinishThreadPool_two") public ThreadPoolExecutor reportFinishThreadPool_two() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - } - - @Bean(name = "reportFinishThreadPool_three") - public ThreadPoolExecutor reportFinishThreadPool_three() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-two-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @@ -212,21 +204,14 @@ public ThreadPoolExecutor reportMachineThreadPool() { @Bean(name = "reportMachineThreadPool_one") public ThreadPoolExecutor reportMachineThreadPool_one() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-one-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "reportMachineThreadPool_two") public ThreadPoolExecutor reportMachineThreadPool_two() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - } - - @Bean(name = "reportMachineThreadPool_three") - public ThreadPoolExecutor reportMachineThreadPool_three() { - ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-two-%d").build(); return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java index cc2b765ac8..83cd56db82 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java @@ -3,10 +3,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.google.common.collect.Maps; import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; @@ -38,6 +40,8 @@ public class SyncMachineDataJob extends AbstractSceneTask implements SimpleJob { private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); + private Map syncMacheineMap = Maps.newHashMap(); + @Override public void execute(ShardingContext shardingContext) { try { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java index c2e1ec0e5c..032b3383d1 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java @@ -82,7 +82,15 @@ public static void main(String[] args) { public ReportCountDTO getReportCount(Long reportId) { ReportSummaryResult data = reportSummaryDAO.selectOneByReportId(reportId); if (data == null) { - return new ReportCountDTO(); + ReportCountDTO dto = new ReportCountDTO(); + dto.setRiskMachineCount(0); + dto.setApplicationCount(0); + dto.setMachineCount(0); + dto.setWarnCount(0); + dto.setBottleneckInterfaceCount(0); + dto.setBusinessActivityCount(0); + dto.setNotpassBusinessActivityCount(0); + return dto; } return convert2ReportCountDTO(data); } @@ -427,7 +435,7 @@ private String[] getTpsConfigLength(List configs) { continue; } - if(min == null || min.length > array.getTime().length) { + if (min == null || min.length > array.getTime().length) { min = array.getTime(); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java index 5c9a3ae745..a33a9d69bf 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java @@ -28,6 +28,7 @@ import io.shulie.takin.web.ext.entity.tenant.TenantCommonExt; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; @@ -99,14 +100,20 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { // 查询报告状态 final ReportDetailOutput report = reportService.getReportById(reportId); if (report == null) { + log.warn("未获取到报告信息,reportId=" + reportId); return false; } // 加锁 // 分布式锁 String lockKey = JobRedisUtils.getRedisJobReport(WebPluginUtils.traceTenantId(), WebPluginUtils.traceEnvCode(), reportId); if (!distributedLock.checkLock(lockKey)) { - // 收集数据 单独线程收集 - threadPoolUtil.getCollectDataThreadPool().execute(collectData(reportId, commonExt, lockKey)); + try { + // 收集数据 单独线程收集 + threadPoolUtil.getCollectDataThreadPool().execute(() -> collectData(reportId, commonExt, lockKey)); + } catch (Throwable e) { + // TODO 如果线程池满了，继续走下面的逻辑,否则任务有问题 + log.error("提交线程池任务异常," + ExceptionUtils.getStackTrace(e)); + } } // 压测结束才锁报告 Integer status = report.getTaskStatus(); @@ -162,7 +169,7 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { reportDataCache.clearDataCache(reportId); log.info("报告id={}汇总成功，花费时间={}", reportId, (System.currentTimeMillis() - startTime)); - } catch (Exception e) { + } catch (Throwable e) { // log.error("客户端生成报告id={}数据异常:{}", reportId, e.getMessage(), e); //生成报告异常，清空本轮生成表数据 reportClearService.clearReportData(reportId); @@ -216,14 +223,14 @@ private void removeReportKey(Long reportId, TenantCommonExt commonExt) { * @param reportId 报告 id * @return 可运行 */ - private synchronized Runnable collectData(Long reportId, TenantCommonExt commonExt, String lockKey) { - return () -> { - boolean tryLock = distributedLock.tryLock(lockKey, 10L, 10L, TimeUnit.SECONDS); + private void collectData(Long reportId, TenantCommonExt commonExt, String lockKey) { + // 有锁,证明任务在处理,不需要等太久,后续会有任务继续处理 + boolean tryLock = distributedLock.tryLock(lockKey, 5L, 60L, TimeUnit.SECONDS); + try { if (!tryLock) { return; } WebPluginUtils.setTraceTenantContext(commonExt); - try { // 检查风险机器 problemAnalysisService.checkRisk(reportId); @@ -242,8 +249,12 @@ private synchronized Runnable collectData(Long reportId, TenantCommonExt commonE } catch (Exception e) { log.error("reportId = {}: total report ,errorMsg= {}", reportId, e.getMessage()); } + } catch (Throwable e) { + log.error("collectData is fail " + ExceptionUtils.getStackTrace(e)); + } finally { distributedLock.unLockSafely(lockKey); - }; + } + } @Override diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java index cf8aeadc7d..4b0bf06644 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java @@ -24,26 +24,22 @@ public class ThreadPoolUtil implements ApplicationContextAware { List syncMachineList = Arrays.asList("reportMachineThreadPool", "reportMachineThreadPool_one", - "reportMachineThreadPool_two", - "reportMachineThreadPool_three"); + "reportMachineThreadPool_two"); List reportTpsList = Arrays.asList("reportTpsThreadPool", "reportTpsThreadPool_one", - "reportTpsThreadPool_two", - "reportTpsThreadPool_three"); + "reportTpsThreadPool_two"); List reportSummaryList = Arrays.asList("reportSummaryThreadPool", "reportSummaryThreadPool_one", - "reportSummaryThreadPool_two", - "reportSummaryThreadPool_three"); + "reportSummaryThreadPool_two"); List reportFinishList = Arrays.asList("reportFinishThreadPool", "reportFinishThreadPool_one", - "reportFinishThreadPool_two", - "reportFinishThreadPool_three"); + "reportFinishThreadPool_two"); List collectDataList = Arrays.asList("collectDataThreadPool", @@ -54,6 +50,8 @@ public class ThreadPoolUtil implements ApplicationContextAware { private static AtomicLong atomicMachine = new AtomicLong(0); private static AtomicLong atomicTps = new AtomicLong(0); private static AtomicLong atomicSummary = new AtomicLong(0); + private static AtomicLong atomicFinish = new AtomicLong(0); + private static AtomicLong atomicCollectData = new AtomicLong(0); static List syncMachineDataJobThreadPools = new ArrayList<>(); static List reportTpsThreadPools = new ArrayList<>(); @@ -102,10 +100,10 @@ public static ThreadPoolExecutor getReportSummaryThreadPool() { } public static ThreadPoolExecutor getReportFinishThreadPool() { - return reportFinishThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % reportFinishThreadPools.size())); + return reportFinishThreadPools.get((int) Math.abs(atomicFinish.getAndIncrement() % reportFinishThreadPools.size())); } public static ThreadPoolExecutor getCollectDataThreadPool() { - return collectDataThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % collectDataThreadPools.size())); + return collectDataThreadPools.get((int) Math.abs(atomicCollectData.getAndIncrement() % collectDataThreadPools.size())); } } From 77e4a2ce16ab245b1429a7110fecb4434c61d819 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 14:45:20 +0800 Subject: [PATCH 28/84] =?UTF-8?q?=E4=B8=AD=E9=97=B4=E4=BB=B6=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E5=8A=A0=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...AgentPushMiddlewareAndCompareConsumer.java | 20 ++++++--- .../ApplicationMiddlewareServiceImpl.java | 42 +++++++++++-------- .../web/biz/threadpool/ThreadPoolUtil.java | 4 +- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java index 84db709517..4cdde8e008 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java @@ -18,6 +18,7 @@ import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.connection.Message; import org.springframework.data.redis.connection.MessageListener; @@ -39,6 +40,10 @@ public class AgentPushMiddlewareAndCompareConsumer implements MessageListener { @Autowired private ApplicationMiddlewareDAO applicationMiddlewareDAO; + //默认不处理中间件信息 + @Value("${takin.enable.middlewareFlag:false}") + private boolean middlewareFlag; + @Autowired @Lazy private ApplicationMiddlewareService applicationMiddlewareService; @@ -46,6 +51,9 @@ public class AgentPushMiddlewareAndCompareConsumer implements MessageListener { @Transactional(rollbackFor = Throwable.class) @Override public void onMessage(Message message, byte[] pattern) { + if (!middlewareFlag) { + return; + } String messageBody = new String(message.getBody()); if (StringUtils.isEmpty(messageBody)) { return; @@ -53,7 +61,7 @@ public void onMessage(Message message, byte[] pattern) { messageBody = messageBody.substring(1, messageBody.length() - 1).replace("\\", ""); MqApplicationMiddlewareCompareDTO mqApplicationMiddlewareCompareDTO = JsonUtil.json2Bean(messageBody, - MqApplicationMiddlewareCompareDTO.class); + MqApplicationMiddlewareCompareDTO.class); if (mqApplicationMiddlewareCompareDTO == null) { return; } @@ -69,15 +77,15 @@ public void onMessage(Message message, byte[] pattern) { try { TenantCommonExt tenantCommonExt = new TenantCommonExt(mqApplicationMiddlewareCompareDTO.getTenantId(), - null, mqApplicationMiddlewareCompareDTO.getEnvCode(), - null, ContextSourceEnum.JOB.getCode()); + null, mqApplicationMiddlewareCompareDTO.getEnvCode(), + null, ContextSourceEnum.JOB.getCode()); WebPluginUtils.setTraceTenantContext(tenantCommonExt); // 根据 applicationId 查询应用中间件 log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件查询"); PageUtils.clearPageHelper(); List applicationMiddlewareList = - applicationMiddlewareDAO.listByApplicationId(applicationId); + applicationMiddlewareDAO.listByApplicationId(applicationId); if (applicationMiddlewareList.isEmpty()) { return; } @@ -85,7 +93,7 @@ public void onMessage(Message message, byte[] pattern) { // 比对 log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件比对"); List updateParamList = - applicationMiddlewareService.doCompare(applicationMiddlewareList); + applicationMiddlewareService.doCompare(applicationMiddlewareList); log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件更新"); applicationMiddlewareDAO.updateBatchById(updateParamList); @@ -103,7 +111,7 @@ public void onMessage(Message message, byte[] pattern) { * @return 相应数据 */ private MqApplicationMiddlewareCompareDTO getMqApplicationMiddlewareCompareDTO( - Message message) { + Message message) { String messageBody = new String(message.getBody()); if (StringUtils.isEmpty(messageBody)) { return null; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java index 32f2281434..af459df30d 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java @@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -70,9 +71,13 @@ public class ApplicationMiddlewareServiceImpl implements ApplicationMiddlewareSe @Autowired private MiddlewareJarService middlewareJarService; + //默认不处理中间件信息 + @Value("${takin.enable.middlewareFlag:false}") + private boolean middlewareFlag; + @Override public PagingList page( - ListApplicationMiddlewareRequest listApplicationMiddlewareRequest) { + ListApplicationMiddlewareRequest listApplicationMiddlewareRequest) { PageApplicationMiddlewareParam pageApplicationMiddlewareParam = new PageApplicationMiddlewareParam(); BeanUtils.copyProperties(listApplicationMiddlewareRequest, pageApplicationMiddlewareParam); @@ -90,7 +95,7 @@ public PagingList page( // 状态转换 ApplicationMiddlewareStatusEnum applicationMiddlewareStatusEnum = - ApplicationMiddlewareStatusEnum.getByCode(result.getStatus()); + ApplicationMiddlewareStatusEnum.getByCode(result.getStatus()); if (applicationMiddlewareStatusEnum != null) { response.setStatusDesc(applicationMiddlewareStatusEnum.getDesc()); } @@ -104,7 +109,7 @@ public PagingList page( public ApplicationMiddlewareCountResponse countSome(Long applicationId) { PageUtils.clearPageHelper(); List statusMapCountResultList = applicationMiddlewareDAO - .listCountByApplicationIdAndStatusAndGroupByStatus(applicationId, null); + .listCountByApplicationIdAndStatusAndGroupByStatus(applicationId, null); if (statusMapCountResultList.isEmpty()) { return new ApplicationMiddlewareCountResponse(); @@ -112,8 +117,8 @@ public ApplicationMiddlewareCountResponse countSome(Long applicationId) { // 状态统计转为 状态 -> 统计个数 map Map statusAboutCount = statusMapCountResultList.stream() - .collect(Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, - ApplicationMiddlewareStatusAboutCountResult::getCount, (v1, v2) -> v2)); + .collect(Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, + ApplicationMiddlewareStatusAboutCountResult::getCount, (v1, v2) -> v2)); ApplicationMiddlewareCountResponse response = new ApplicationMiddlewareCountResponse(); response.setTotalCount(applicationMiddlewareDAO.countByApplicationIdAndStatus(applicationId, null)); @@ -122,7 +127,7 @@ public ApplicationMiddlewareCountResponse countSome(Long applicationId) { response.setNotSupportedCount(statusAboutCount.get(ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode())); response.setNoneCount(statusAboutCount.get(ApplicationMiddlewareStatusEnum.NONE.getCode())); response.setNoSupportRequiredCount( - statusAboutCount.get(ApplicationMiddlewareStatusEnum.NO_SUPPORT_REQUIRED.getCode())); + statusAboutCount.get(ApplicationMiddlewareStatusEnum.NO_SUPPORT_REQUIRED.getCode())); return response; } @@ -156,7 +161,7 @@ public void compare(Long applicationId) { public List doCompare(List results) { // 转 dto List compareApplicationMiddlewareList = DataTransformUtil.list2list(results, - CompareApplicationMiddlewareDTO.class); + CompareApplicationMiddlewareDTO.class); // 比对 middlewareJarService.appCompare(compareApplicationMiddlewareList); @@ -168,6 +173,9 @@ public List doCompare(List middlewareList = pushMiddlewareRequest.getMiddlewareList(); if (middlewareList.isEmpty()) { return; @@ -195,9 +203,9 @@ public void pushMiddlewareList(PushMiddlewareRequest pushMiddlewareRequest) { // 新的中间件插入 log.info("应用中间件上报 --> 插入上报中间件"); List createApplicationMiddlewareParamList = - this.listCreateApplicationMiddlewareParam(middlewareList, application); + this.listCreateApplicationMiddlewareParam(middlewareList, application); this.isPushError(!applicationMiddlewareDAO.insertBatch(createApplicationMiddlewareParamList), - "应用中间件报错失败!"); + "应用中间件报错失败!"); } catch (Exception e) { // 发生错误, 解锁 @@ -221,20 +229,20 @@ public void pushMiddlewareList(PushMiddlewareRequest pushMiddlewareRequest) { @Override public Map> getApplicationNameAboutStatusCountMap( - List applicationIds) { + List applicationIds) { List statusList = Arrays.asList(ApplicationMiddlewareStatusEnum.NONE.getCode(), - ApplicationMiddlewareStatusEnum.UNKNOWN.getCode(), - ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode()); + ApplicationMiddlewareStatusEnum.UNKNOWN.getCode(), + ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode()); List results = applicationMiddlewareDAO - .listStatusCountByAndGroupByApplicationNameListAndStatus(applicationIds, statusList); + .listStatusCountByAndGroupByApplicationNameListAndStatus(applicationIds, statusList); if (results.isEmpty()) { return Collections.emptyMap(); } return results.stream() - .collect(Collectors.groupingBy(ApplicationMiddlewareStatusAboutCountResult::getApplicationName, - Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, - ApplicationMiddlewareStatusAboutCountResult::getCount))); + .collect(Collectors.groupingBy(ApplicationMiddlewareStatusAboutCountResult::getApplicationName, + Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, + ApplicationMiddlewareStatusAboutCountResult::getCount))); } /** @@ -267,7 +275,7 @@ private void isPushError(boolean condition, String message) { * @return 待新增的中间件对象列表 */ private List listCreateApplicationMiddlewareParam( - List middlewareList, ApplicationDetailResult application) { + List middlewareList, ApplicationDetailResult application) { return middlewareList.stream().map(pushMiddlewareListRequest -> { CreateApplicationMiddlewareParam createParam = new CreateApplicationMiddlewareParam(); createParam.setApplicationId(application.getApplicationId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java index 4b0bf06644..fae6be626f 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java @@ -13,7 +13,9 @@ import java.util.concurrent.atomic.AtomicLong; /** - * @author xingchen + * 隔离任务线程池,相互之间不影响，彼此之间不影响 + * + * @author * @description: TODO * @date 2022/7/7 9:32 AM */ From e7fa4c50c6cabd4fc73e6a9959660ac2af32298e Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 16:35:37 +0800 Subject: [PATCH 29/84] =?UTF-8?q?=E7=99=BD=E5=90=8D=E5=8D=95-=E5=BA=94?= =?UTF-8?q?=E7=94=A8=E5=90=AF=E5=8A=A8=E7=9A=84=E6=97=B6=E5=80=99=E4=B8=8D?= =?UTF-8?q?=E8=A6=81=E5=88=9D=E5=A7=8B=E5=8C=96=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/biz/service/ConfCenterService.java | 9 +++ .../linkmanage/impl/WhiteListFileService.java | 56 +++++++++++-------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java index ef0cb9b7ad..1ff7578fef 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java @@ -99,6 +99,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -150,6 +151,10 @@ public class ConfCenterService extends CommonService { public static final String APPLICATION_CACHE_PREFIX = "application:cache"; + // 是否默认初始化白名单 + @Value("${takin.enable.initWhiteList:false}") + private boolean initWhiteList; + @PostConstruct public void init() { number = ConfigServerHelper.getWrapperIntegerValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_NUMBER_LIMIT); @@ -244,6 +249,10 @@ private void addApplicationToDataBuild(ApplicationCreateParam tApplicationMnt) { @PostConstruct public void initWhiteList() { + if (!initWhiteList) { + log.info("不初始化白名单到文件"); + return; + } writeWhiteListFile(); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java index 2a76dda0ac..4f716d66c8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java @@ -79,13 +79,21 @@ public class WhiteListFileService { @Autowired private WhiteListService whiteListService; + // 是否默认初始化白名单 + @Value("${takin.enable.initWhiteList:false}") + private boolean initWhiteList; + @PostConstruct public void init() { + if (!initWhiteList) { + log.info("不初始化白名单到文件"); + return; + } ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( - 0, 1, - 0, TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(1), - r -> new Thread(r, "初始化白名单"), new CallerRunsPolicy()); + 0, 1, + 0, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1), + r -> new Thread(r, "初始化白名单"), new CallerRunsPolicy()); threadPoolExecutor.submit(() -> { log.info("开始初始化白名单"); // 老版本 agent 新版本agent 已转到远程调用模块 @@ -173,10 +181,10 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e Map> whitelistMap; boolean isCheckDuplicateName = Boolean.parseBoolean( - ConfigServerHelper.getValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK)); + ConfigServerHelper.getValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK)); if (isCheckDuplicateName) { List armdString = agentWhiteLists.stream().map(AgentWhiteList::getInterfaceName).collect( - Collectors.toList()); + Collectors.toList()); existWhite = whiteListService.getExistWhite(armdString, Lists.newArrayList()); // todo 这里再获取一次，感觉很多余，但是不改上面的逻辑，所有这里数据再次从新获取，之后可以重构下 WhitelistSearchParam param = new WhitelistSearchParam(); @@ -184,7 +192,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e param.setUseYn(1); List results = whiteListDAO.getList(param); whitelistMap = results.stream().collect( - Collectors.groupingBy(e -> e.getInterfaceName() + "@@" + e.getType())); + Collectors.groupingBy(e -> e.getInterfaceName() + "@@" + e.getType())); } else { // 获取所有白名单，是否有全局属性 WhitelistSearchParam param = new WhitelistSearchParam(); @@ -193,7 +201,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e param.setUseYn(1); List results = whiteListDAO.getList(param); whitelistMap = results.stream() - .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); + .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); } // 获取所有生效效应，是否有局部应用 @@ -203,7 +211,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e searchParam.setWlistIds(ids); List appResults = whitelistEffectiveAppDao.getList(searchParam); Map> appResultsMap = appResults.stream() - .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); + .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); List> wListsResult = Lists.newArrayList(); if (CollectionUtils.isNotEmpty(agentWhiteLists)) { @@ -242,9 +250,9 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e //生效应用 List appLists = appResultsMap.get(id); whiteItemNew.put("appNames", CollectionUtils.isNotEmpty(appLists) ? - appLists.stream().map(WhitelistEffectiveAppResult::getEffectiveAppName).distinct() - .collect(Collectors.toList()) - : Lists.newArrayList()); + appLists.stream().map(WhitelistEffectiveAppResult::getEffectiveAppName).distinct() + .collect(Collectors.toList()) + : Lists.newArrayList()); wListsResult.add(whiteItemNew); } } @@ -261,7 +269,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e private List> getBlackList(TenantCommonExt tenantCommonExt) { List tbLists = tbListMntDao.getAllEnabledBlockList(tenantCommonExt.getTenantId(), - tenantCommonExt.getEnvCode()); + tenantCommonExt.getEnvCode()); if (CollectionUtils.isEmpty(tbLists)) { return Lists.newArrayList(); } @@ -283,10 +291,10 @@ private List getNewBlackList(TenantCommonExt tenantCommonExt) List detailResults = applicationDAO.getApplicationList(param); Map> redisMap = results.stream() - .collect(Collectors.groupingBy(BlacklistResult::getApplicationId)); + .collect(Collectors.groupingBy(BlacklistResult::getApplicationId)); Map> detailResultMap = detailResults.stream() - .collect(Collectors.groupingBy(ApplicationDetailResult::getApplicationId)); + .collect(Collectors.groupingBy(ApplicationDetailResult::getApplicationId)); List vos = Lists.newArrayList(); for (Long id : redisMap.keySet()) { List app = detailResultMap.get(id); @@ -309,20 +317,20 @@ private List getNewBlackList(TenantCommonExt tenantCommonExt) private List agentListWhitelist(TenantCommonExt ext) { List list = applicationDAO.queryIdsByNameAndTenant(Lists.newArrayList(), - ext != null ? ext.getTenantId() : null, ext != null ? ext.getEnvCode() : null); + ext != null ? ext.getTenantId() : null, ext != null ? ext.getEnvCode() : null); if (CollectionUtils.isEmpty(list)) { return Lists.newArrayList(); } List> maps = whiteListDAO.getWhiteListByAppIds(list); return maps.stream().map(it -> { - AgentWhiteList whiteListDTO = new AgentWhiteList(); - whiteListDTO.setInterfaceName((String)it.get("interfaceName")); - whiteListDTO.setType(getType(Integer.parseInt((String)it.get("type")))); - whiteListDTO.setSourceType((String)it.get("type")); - // 过滤生效应用用 - whiteListDTO.setWlistId((Long)it.get("wlistId")); - return whiteListDTO; - } + AgentWhiteList whiteListDTO = new AgentWhiteList(); + whiteListDTO.setInterfaceName((String) it.get("interfaceName")); + whiteListDTO.setType(getType(Integer.parseInt((String) it.get("type")))); + whiteListDTO.setSourceType((String) it.get("type")); + // 过滤生效应用用 + whiteListDTO.setWlistId((Long) it.get("wlistId")); + return whiteListDTO; + } ).collect(Collectors.toList()); } From 4446bd9406c5fe34cc09c4066710d0bec5e5069d Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 7 Jul 2022 16:54:20 +0800 Subject: [PATCH 30/84] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=B8=B4=E6=97=B6?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/entrypoint/controller/file/FileController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/takin-web-entrypoint/src/main/java/io/shulie/takin/web/entrypoint/controller/file/FileController.java b/takin-web-entrypoint/src/main/java/io/shulie/takin/web/entrypoint/controller/file/FileController.java index 7fc84175e6..5c83f6f161 100644 --- a/takin-web-entrypoint/src/main/java/io/shulie/takin/web/entrypoint/controller/file/FileController.java +++ b/takin-web-entrypoint/src/main/java/io/shulie/takin/web/entrypoint/controller/file/FileController.java @@ -82,9 +82,11 @@ public List upload(List file) { throw new RuntimeException("上传文件不能为空"); } } - return cloudFileApi.upload(new UploadRequest() {{ + List response = cloudFileApi.upload(new UploadRequest() {{ setFileList(FileUtil.convertMultipartFileList(file)); }}); + FileUtil.deleteTempFile(file); + return response; } @PostMapping("/attachment/upload") From 1b3f6070c9b1bef47bf0c3d10e6f625d1d803a83 Mon Sep 17 00:00:00 2001 From: nanfeng Date: Fri, 8 Jul 2022 10:55:26 +0800 Subject: [PATCH 31/84] =?UTF-8?q?agent=E6=8B=89=E5=8F=96=E5=BD=B1=E5=AD=90?= =?UTF-8?q?job=E7=9A=84=E6=9F=A5=E8=AF=A2=E9=99=A4=E5=8E=BB=E6=9D=83?= =?UTF-8?q?=E9=99=90id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/biz/service/simplify/ShadowJobConfigService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java index ebd1c9123e..4e3b4f68f1 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java @@ -260,7 +260,7 @@ public List queryByAppName(String appName) { query.setOrderBy("id desc"); } query.setApplicationId(tApplicationMnt.getApplicationId()); - query.setUserIds(WebPluginUtils.getQueryAllowUserIdList()); +// query.setUserIds(WebPluginUtils.getQueryAllowUserIdList()); //agent调用不需要 List tShadowJobConfigs = tShadowJobConfigMapper.selectList(query); try { for (TShadowJobConfig tShadowJobConfig : tShadowJobConfigs) { From b6622e34f496f3efd054bb55ac709d13a4844c94 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Tue, 12 Jul 2022 16:08:56 +0800 Subject: [PATCH 32/84] merge from bugfix/5.5.0 --- .../service/impl/ApplicationServiceImpl.java | 64 +++--- .../impl/ShadowConsumerServiceImpl.java | 189 +++++++++--------- 2 files changed, 133 insertions(+), 120 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 823c8bf4e1..fb13af54b8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -744,7 +744,7 @@ public synchronized void syncApplicationAccessStatus() { } - this.syncApplicationAccessStatus(applicationList,errorApplicationIdSet); + this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet); } while (applicationNumber == pageSize); // 先执行一遍, 然后如果分页应用数量等于pageSize, 那么查询下一页 @@ -754,18 +754,22 @@ public synchronized void syncApplicationAccessStatus() { log.debug("定时同步应用状态完成!"); } - private void syncApplicationAccessStatus(List applicationList,Set errorApplicationIdSet) { + private void syncApplicationAccessStatus(List applicationList, Set errorApplicationIdSet) { if (CollectionUtils.isNotEmpty(applicationList)) { - applicationList.forEach(app -> { + for (ApplicationListResult app : applicationList) { Map result = applicationDAO.getStatus(app.getApplicationName()); long n = (long) result.get("n"); if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = (String) result.get("e"); + //不知道异常和Ip就别展示出来误导了 if (StringUtils.isBlank(e)) { - String a = (String)result.get("a"); - e = "探针接入异常"; - if (StringUtils.isNotEmpty(a)) { - e += "，agentId为"+a; + String a = (String) result.get("a"); +// e = "探针接入异常"; +// if (StringUtils.isNotEmpty(a)) { +// e += "，agentId为" + a; +// } + if (StringUtils.isEmpty(a)) { + continue; } } applicationDAO.updateStatus(app.getApplicationId(), e); @@ -783,8 +787,10 @@ private void syncApplicationAccessStatus(List application param.setSwitchErrorMap(map); uploadAccessStatus(param); } else { - applicationDAO.updateStatus(app.getApplicationId());} - }); + applicationDAO.updateStatus(app.getApplicationId()); + } + + } } } @@ -1187,8 +1193,8 @@ public String getApplicationNameByApplicationId(Long applicationId) { @Override public void uninstallAllAgent(List appIds) { try { - appIds = this.filterAppIds(appIds,AgentConstants.UNINSTALL); - if (CollectionUtils.isEmpty(appIds)){ + appIds = this.filterAppIds(appIds, AgentConstants.UNINSTALL); + if (CollectionUtils.isEmpty(appIds)) { log.info("所有需要卸载的应用都被过滤掉了"); return; } @@ -1341,7 +1347,7 @@ public PagingList listApplicationByUpgrade(App @Override public Response operateCheck(List appIds, String operate) { - if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)){ + if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)) { return Response.fail("参数异常"); } @@ -1357,25 +1363,25 @@ public Response operateCheck(List appIds, String operate) { List appNames = applicationList.stream().map(ApplicationDetailResult::getApplicationName).collect( Collectors.toList()); List applicationNodeProbeResults = applicationNodeProbeDAO.listByAppNameAndOperate(ApplicationNodeProbeOperateEnum.UNINSTALL.getCode(), appNames); - long count = applicationNodeProbeResults == null ? 0 : applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().count(); - if (AgentConstants.UNINSTALL.equals(operate)){ - if (count > 0){ + long count = applicationNodeProbeResults == null ? 0 : applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().count(); + if (AgentConstants.UNINSTALL.equals(operate)) { + if (count > 0) { //构建返回数据 List distinct = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().collect(Collectors.toList()); StringBuilder sb = new StringBuilder(); distinct.forEach(s -> { sb.append(s).append("\n"); }); - return Response.success(String.format("已选择%d个应用,%d个应用已处于卸载状态\n应用名称为:",appIds.size(),count) + sb); - }else { - return Response.success(String.format("已选择%d个应用,点击继续卸载",appIds.size())); + return Response.success(String.format("已选择%d个应用,%d个应用已处于卸载状态\n应用名称为:", appIds.size(), count) + sb); + } else { + return Response.success(String.format("已选择%d个应用,点击继续卸载", appIds.size())); } } - if (AgentConstants.RESUME.equals(operate)){ - if (appIds.size() > count){ + if (AgentConstants.RESUME.equals(operate)) { + if (appIds.size() > count) { //构建返回数据 List result = appNames; - if (count != 0){ + if (count != 0) { List distinct = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().collect(Collectors.toList()); result = result.stream().filter(o -> !distinct.contains(o)).collect(Collectors.toList()); } @@ -1383,9 +1389,9 @@ public Response operateCheck(List appIds, String operate) { result.forEach(s -> { sb.append(s).append("\n"); }); - return Response.success(String.format("已选择%d个应用,%d个应用处于非卸载状态\n应用名称为:",appIds.size(), appIds.size() - count) + sb); - }else { - return Response.success(String.format("已选择%d个应用,点击继续",appIds.size())); + return Response.success(String.format("已选择%d个应用,%d个应用处于非卸载状态\n应用名称为:", appIds.size(), appIds.size() - count) + sb); + } else { + return Response.success(String.format("已选择%d个应用,点击继续", appIds.size())); } } return Response.fail("上传的状态当前不支持校验"); @@ -1393,7 +1399,7 @@ public Response operateCheck(List appIds, String operate) { @Override public List filterAppIds(List appIds, String operate) { - if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)){ + if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)) { return null; } @@ -1413,12 +1419,12 @@ public List filterAppIds(List appIds, String operate) { List uninstallAppNames = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName) .collect(Collectors.toList()); //需要卸载的数据，需要不存在卸载的数据 - if (AgentConstants.UNINSTALL.equals(operate)){ + if (AgentConstants.UNINSTALL.equals(operate)) { return applicationList.stream().filter(o -> !uninstallAppNames.contains(o.getApplicationName())).map(o -> o.getApplicationId().toString()).collect(Collectors.toList()); } //需要恢复的数据，需要是已经卸载的数据 - if (AgentConstants.RESUME.equals(operate)){ + if (AgentConstants.RESUME.equals(operate)) { return applicationList.stream().filter(o -> uninstallAppNames.contains(o.getApplicationName())).map(o -> o.getApplicationId().toString()).collect(Collectors.toList()); } @@ -2449,8 +2455,8 @@ public Response uploadMiddlewareStatus(Map reques @Override public void resumeAllAgent(List appIds) { try { - appIds = this.filterAppIds(appIds,AgentConstants.RESUME); - if (CollectionUtils.isEmpty(appIds)){ + appIds = this.filterAppIds(appIds, AgentConstants.RESUME); + if (CollectionUtils.isEmpty(appIds)) { log.info("所有需要恢复的应用都被过滤掉了"); return; } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java index 6381118ef4..dcf22202f8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java @@ -1,12 +1,7 @@ package io.shulie.takin.web.biz.service.impl; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Resource; @@ -137,7 +132,7 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput ApplicationDetailResult application = applicationDAO.getApplicationById(request.getApplicationId()); if (application == null) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, - String.format("应用id:%s对应的应用不存在", request.getApplicationId())); + String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); if (StringUtils.isNotBlank(request.getTopicGroup())) { @@ -148,7 +143,7 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput } if (request.getEnabled() != null) { lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getStatus, - request.getEnabled() ? ShadowConsumerConstants.ENABLE : ShadowConsumerConstants.DISABLE); + request.getEnabled() ? ShadowConsumerConstants.ENABLE : ShadowConsumerConstants.DISABLE); } if (CollectionUtils.isNotEmpty(WebPluginUtils.getQueryAllowUserIdList())) { lambdaQueryWrapper.in(ShadowMqConsumerEntity::getUserId, WebPluginUtils.getQueryAllowUserIdList()); @@ -167,72 +162,72 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput } private List filterResult(ShadowConsumerQueryInput request, - List totalResult) { + List totalResult) { if (request.getEnabled() != null) { if (request.getEnabled()) { totalResult = totalResult.stream().filter(ShadowConsumerOutput::getEnabled).collect( - Collectors.toList()); + Collectors.toList()); } else { totalResult = totalResult.stream().filter(e -> !e.getEnabled()).collect(Collectors.toList()); } } if (StringUtils.isNotBlank(request.getTopicGroup())) { totalResult = totalResult.stream().filter(e -> e.getTopicGroup().contains(request.getTopicGroup())).collect( - Collectors.toList()); + Collectors.toList()); } if (request.getType() != null) { totalResult = totalResult.stream().filter(e -> e.getType().equals(request.getType())).collect( - Collectors.toList()); + Collectors.toList()); } return totalResult; } private List mergeResult(List amdbResult, - List dbResult) { + List dbResult) { Map amdbMap = new HashMap<>(); Map entityMap = mqConfigTemplateDAO.selectToMapWithNameKey(); if (CollectionUtils.isNotEmpty(amdbResult)) { amdbMap = amdbResult.stream() - .filter(item -> entityMap.containsKey(item.getType())) - .map(e -> { - ShadowConsumerOutput response = new ShadowConsumerOutput(); - response.setUnionId( - MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); - response.setType(e.getType()); - response.setTopicGroup(e.getTopicGroup()); - response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); - response.setGmtCreate(e.getCreateTime()); - response.setGmtUpdate(e.getUpdateTime()); - response.setCanRemove(false); - response.setCanEnableDisable(false); - response.setIsManual(false); - response.setShadowconsumerEnable(String.valueOf(e.getStatus())); - return response; - }) - .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); + .filter(item -> entityMap.containsKey(item.getType())) + .map(e -> { + ShadowConsumerOutput response = new ShadowConsumerOutput(); + response.setUnionId( + MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); + response.setType(e.getType()); + response.setTopicGroup(e.getTopicGroup()); + response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); + response.setGmtCreate(e.getCreateTime()); + response.setGmtUpdate(e.getUpdateTime()); + response.setCanRemove(false); + response.setCanEnableDisable(false); + response.setIsManual(false); + response.setShadowconsumerEnable(String.valueOf(e.getStatus())); + return response; + }) + .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); } Map dbMap = new HashMap<>(dbResult.size()); if (CollectionUtils.isNotEmpty(dbResult)) { dbMap = dbResult.stream() - .filter(item -> entityMap.containsKey(item.getType())) - .map(e -> { - ShadowConsumerOutput response = new ShadowConsumerOutput(); - response.setId(e.getId()); - response.setUnionId( - MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); - response.setType(e.getType()); - response.setTopicGroup(e.getTopicGroup()); - response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); - response.setGmtCreate(e.getCreateTime()); - response.setGmtUpdate(e.getUpdateTime()); - response.setUserId(e.getUserId()); - response.setIsManual(e.getManualTag() == 1); - response.setCanRemove(response.getIsManual()); - response.setShadowconsumerEnable(String.valueOf(e.getStatus())); - WebPluginUtils.fillQueryResponse(response); - return response; - }) - .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); + .filter(item -> entityMap.containsKey(item.getType())) + .map(e -> { + ShadowConsumerOutput response = new ShadowConsumerOutput(); + response.setId(e.getId()); + response.setUnionId( + MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); + response.setType(e.getType()); + response.setTopicGroup(e.getTopicGroup()); + response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); + response.setGmtCreate(e.getCreateTime()); + response.setGmtUpdate(e.getUpdateTime()); + response.setUserId(e.getUserId()); + response.setIsManual(e.getManualTag() == 1); + response.setCanRemove(response.getIsManual()); + response.setShadowconsumerEnable(String.valueOf(e.getStatus())); + WebPluginUtils.fillQueryResponse(response); + return response; + }) + .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); } // 原：在amdb自动梳理的基础上，补充数据库里面的记录，有的话用数据的记录 // 现：在db的基础上，补充amdb自动梳理的数据。 @@ -248,39 +243,51 @@ private List mergeResult(List amdb } private List queryAmdbDefaultEntrances(ShadowConsumerQueryInput request, - String applicationName) { + String applicationName) { List mqTopicGroups = applicationEntranceClient.getMqTopicGroups(applicationName); if (CollectionUtils.isEmpty(mqTopicGroups)) { return Lists.newArrayList(); } if (Objects.nonNull(request.getType())) { mqTopicGroups = mqTopicGroups.stream() - .filter(dto -> dto.getMiddlewareName().equals(request.getType())) - .collect(Collectors.toList()); + .filter(dto -> dto.getMiddlewareName().equals(request.getType())) + .collect(Collectors.toList()); } return mqTopicGroups.stream() - .map(mqTopicGroup -> { - ShadowMqConsumerOutput shadowMqConsumerOutput = new ShadowMqConsumerOutput(); - shadowMqConsumerOutput.setTopicGroup( - mqTopicGroup.getServiceName() + "#" + mqTopicGroup.getMethodName()); - shadowMqConsumerOutput.setType( - MiddlewareTypeGroupEnum.getMiddlewareGroupType(mqTopicGroup.getMiddlewareName()).getType()); - shadowMqConsumerOutput.setApplicationId(request.getApplicationId()); - shadowMqConsumerOutput.setApplicationName(applicationName); - shadowMqConsumerOutput.setStatus(ShadowConsumerConstants.DISABLE); - shadowMqConsumerOutput.setDeleted(ShadowConsumerConstants.LIVED); - // 补充数据 - WebPluginUtils.fillUserData(shadowMqConsumerOutput); - return shadowMqConsumerOutput; - }).collect(Collectors.toList()); + .map(mqTopicGroup -> { + ShadowMqConsumerOutput shadowMqConsumerOutput = new ShadowMqConsumerOutput(); + shadowMqConsumerOutput.setTopicGroup( + mqTopicGroup.getServiceName() + "#" + mqTopicGroup.getMethodName()); + shadowMqConsumerOutput.setType( + MiddlewareTypeGroupEnum.getMiddlewareGroupType(mqTopicGroup.getMiddlewareName()).getType()); + shadowMqConsumerOutput.setApplicationId(request.getApplicationId()); + shadowMqConsumerOutput.setApplicationName(applicationName); + shadowMqConsumerOutput.setStatus(ShadowConsumerConstants.DISABLE); + shadowMqConsumerOutput.setDeleted(ShadowConsumerConstants.LIVED); + // 补充数据 + WebPluginUtils.fillUserData(shadowMqConsumerOutput); + return shadowMqConsumerOutput; + }).collect(Collectors.toList()); } private PagingList splitPage( - ShadowConsumerQueryInput request, - List responses) { + ShadowConsumerQueryInput request, + List responses) { responses.sort((o1, o2) -> { + boolean o1InValid = o1.getGmtCreate() == null; + boolean o2Invalid = o2.getGmtCreate() == null; + boolean bothInvalid = o1InValid && o2Invalid; + if (bothInvalid) { + return 0; + } + if (o1InValid) { + return -1; + } + if (o2Invalid) { + return 1; + } if (o1.getGmtCreate() != null && o2.getGmtCreate() != null) { int firstSort = -o1.getGmtCreate().compareTo(o2.getGmtCreate()); if (firstSort == 0) { @@ -312,10 +319,10 @@ public void createMqConsumers(ShadowConsumerCreateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.CREATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -349,12 +356,12 @@ public void updateMqConsumers(ShadowConsumerUpdateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); // 同名的自己不算 exists = exists.stream().filter(item -> !item.getId().equals(request.getId())).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.UPDATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -371,7 +378,7 @@ public void updateMqConsumers(ShadowConsumerUpdateInput request) { @Override public void importUpdateMqConsumers(ShadowConsumerUpdateInput request) { if (!request.getTopicGroup().contains("#")) { - return; + return; } String[] split = request.getTopicGroup().split("#"); if (split.length != 2) { @@ -382,7 +389,7 @@ public void importUpdateMqConsumers(ShadowConsumerUpdateInput request) { return; } ShadowMqConsumerEntity updateEntity = new ShadowMqConsumerEntity(); - BeanUtils.copyProperties(request,updateEntity); + BeanUtils.copyProperties(request, updateEntity); updateEntity.setTopicGroup(request.getTopicGroup()); updateEntity.setType(request.getType()); updateEntity.setStatus(request.getStatus()); @@ -452,12 +459,12 @@ public void operateMqConsumers(ShadowConsumersOperateInput requests) { } else { OperationLogContextHolder.operationType(OpTypes.DISABLE); List ids = requests.getRequests().stream().map(ShadowConsumerOperateInput::getId).collect( - Collectors.toList()); + Collectors.toList()); LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); lambdaQueryWrapper.in(ShadowMqConsumerEntity::getId, ids); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getDeleted, ShadowConsumerConstants.LIVED); List shadowMqConsumerEntities = shadowMqConsumerMapper.selectList( - lambdaQueryWrapper); + lambdaQueryWrapper); if (CollectionUtils.isNotEmpty(shadowMqConsumerEntities)) { for (ShadowMqConsumerEntity shadowMqConsumerEntity : shadowMqConsumerEntities) { ShadowMqConsumerEntity updateEntity = new ShadowMqConsumerEntity(); @@ -482,14 +489,14 @@ public List agentSelect(String appName) { return Lists.newArrayList(); } Map> collect = entities.stream() - .filter(t -> { - if (StringUtils.isNotBlank(t.getTopicGroup())) { - String[] topicGroup = t.getTopicGroup().trim().split("#"); - return topicGroup.length == 2; - } else { - return false; - } - }).collect(Collectors.groupingBy(ShadowMqConsumerEntity::getType)); + .filter(t -> { + if (StringUtils.isNotBlank(t.getTopicGroup())) { + String[] topicGroup = t.getTopicGroup().trim().split("#"); + return topicGroup.length == 2; + } else { + return false; + } + }).collect(Collectors.groupingBy(ShadowMqConsumerEntity::getType)); if (MapUtils.isEmpty(collect)) { return Lists.newArrayList(); } @@ -526,7 +533,7 @@ public int allocationUser(ShadowConsumerUpdateUserInput request) { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); queryWrapper.eq(ShadowMqConsumerEntity::getApplicationId, request.getApplicationId()); List shadowMqConsumerEntityList = shadowMqConsumerMapper.selectList( - queryWrapper); + queryWrapper); if (CollectionUtils.isNotEmpty(shadowMqConsumerEntityList)) { for (ShadowMqConsumerEntity entity : shadowMqConsumerEntityList) { entity.setUserId(request.getUserId()); @@ -571,7 +578,7 @@ public List queryMqSupportProgramme(String engName) { @Override @Transactional(rollbackFor = Throwable.class) public void updateMqConsumersV2(ShadowConsumerUpdateInput request) { - request.setTopicGroup(StringUtil.isEmpty(request.getTopicGroup()) ?"":request.getTopicGroup().trim()); + request.setTopicGroup(StringUtil.isEmpty(request.getTopicGroup()) ? "" : request.getTopicGroup().trim()); if (Objects.isNull(request.getId())) { this.createMqConsumersV2(request, false); } else { @@ -587,12 +594,12 @@ public void updateMqConsumersV2(ShadowConsumerUpdateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); // 同名的自己不算 exists = exists.stream().filter(item -> !item.getId().equals(request.getId())).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.UPDATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -623,10 +630,10 @@ public void createMqConsumersV2(ShadowConsumerCreateInput request, Boolean manua throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.CREATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -651,7 +658,7 @@ public PagingList pageMqConsumersV2(ShadowConsumerQueryInp ShadowConsumerQueryInput queryInput = Convert.convert(ShadowConsumerQueryInput.class, request); if (application == null) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, - String.format("应用id:%s对应的应用不存在", request.getApplicationId())); + String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); if (StringUtils.isNotBlank(request.getTopicGroup())) { From a11f856dc574bf978aaa1f80a48d625f593422fe Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Tue, 12 Jul 2022 18:06:15 +0800 Subject: [PATCH 33/84] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/ApplicationErrorServiceImpl.java | 18 +++++++++++------- .../service/impl/ApplicationServiceImpl.java | 8 ++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java index 5315ece1cc..42c3776b51 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java @@ -18,6 +18,7 @@ import com.pamirs.takin.common.util.DateUtils; import com.pamirs.takin.entity.domain.dto.NodeUploadDataDTO; import com.pamirs.takin.entity.domain.entity.ExceptionInfo; +import io.shulie.takin.utils.string.StringUtil; import io.shulie.takin.web.biz.pojo.input.application.ApplicationErrorQueryInput; import io.shulie.takin.web.biz.pojo.output.application.ApplicationErrorOutput; import io.shulie.takin.web.biz.pojo.output.application.ApplicationExceptionOutput; @@ -143,13 +144,16 @@ private void convertNodeUploadDataList(List responseList log.error("异常转换失败：错误信息: {},异常内容{}", message, e.getMessage()); } ApplicationErrorOutput applicationErrorResponse - = new ApplicationErrorOutput() - .setExceptionId(exceptionInfo != null ? exceptionInfo.getErrorCode() : "web-异常原文显示") - .setAgentIdList(Collections.singletonList(nodeUploadDataDTO.getAgentId())) - .setDescription(exceptionInfo != null ? exceptionInfo.getMessage() : message) - .setDetail(exceptionInfo != null ? exceptionInfo.getDetail() : message) - .setTime(nodeUploadDataDTO.getExceptionTime()); - responseList.add(applicationErrorResponse); + = new ApplicationErrorOutput() + .setExceptionId(exceptionInfo != null ? exceptionInfo.getErrorCode() : "web-异常原文显示") + .setAgentIdList(Collections.singletonList(nodeUploadDataDTO.getAgentId())) + .setDescription(exceptionInfo != null ? exceptionInfo.getMessage() : message) + .setDetail(exceptionInfo != null ? exceptionInfo.getDetail() : message) + .setTime(nodeUploadDataDTO.getExceptionTime()); + if (!StringUtil.equals("探针接入异常", applicationErrorResponse.getDetail()) + || !StringUtil.equals("探针接入异常", applicationErrorResponse.getDescription())) { + responseList.add(applicationErrorResponse); + } } } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index fb13af54b8..713ed96204 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -764,13 +764,13 @@ private void syncApplicationAccessStatus(List application //不知道异常和Ip就别展示出来误导了 if (StringUtils.isBlank(e)) { String a = (String) result.get("a"); -// e = "探针接入异常"; -// if (StringUtils.isNotEmpty(a)) { -// e += "，agentId为" + a; -// } if (StringUtils.isEmpty(a)) { continue; } + e = "探针接入异常"; + if (StringUtils.isNotEmpty(a)) { + e += "，agentId为" + a; + } } applicationDAO.updateStatus(app.getApplicationId(), e); NodeUploadDataDTO param = new NodeUploadDataDTO(); From faa25ab737135ad6f9679097b83e3d70d66933a5 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 14 Jul 2022 16:14:29 +0800 Subject: [PATCH 34/84] =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=A4=B1=E6=95=88,?= =?UTF-8?q?=E6=B5=81=E9=87=8F=E9=80=8F=E4=BC=A0=E5=88=B0mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/ClearDsDataConfig.java | 4 +- .../biz/cache/AbstractAgentConfigCache.java | 71 +++++++++++++++++-- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java index 6f8de4419b..7cdd00b5eb 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java @@ -22,7 +22,7 @@ public class ClearDsDataConfig implements ApplicationRunner { @Override public void run(ApplicationArguments args) throws Exception { - log.info("spring启动成功后重新清理缓存"); - cacheList.forEach(AbstractAgentConfigCache::reset); + //log.info("spring启动成功后重新清理缓存"); + //cacheList.forEach(AbstractAgentConfigCache::reset); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index 03320be449..ba6e945fad 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -1,14 +1,20 @@ package io.shulie.takin.web.biz.cache; import java.util.Set; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; +import io.shulie.takin.web.biz.service.DistributedLock; +import io.shulie.takin.web.biz.service.impl.RedissonDistributedLock; import io.shulie.takin.web.common.util.CommonUtil; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; /** @@ -21,6 +27,16 @@ public abstract class AbstractAgentConfigCache implements AgentCacheSupport 50) { + log.warn("已超过递归次数,但还是未获取到值,namespace:", namespace); + return null; + } + String cacheKey = getCacheKey(namespace); + T result = (T) redisTemplate.opsForValue().get(cacheKey); if (result == null) { - result = queryValue(namespace); - redisTemplate.opsForValue().set(getCacheKey(namespace), result,5, TimeUnit.MINUTES); + // 单独走一个查询的分布式key + String queryLockKey = "t:data:query:" + cacheKey; + // 10ms去拿一次,操作很快会完成 + boolean isLock = distributedLock.tryLock(queryLockKey, 20L, 1000L, TimeUnit.MILLISECONDS); + if (isLock) { + try { + result = queryValue(namespace); + redisTemplate.opsForValue().set(cacheKey, result, 5, TimeUnit.MINUTES); + } catch (Throwable e) { + log.error("数据操作失败 " + ExceptionUtils.getStackTrace(e)); + } finally { + distributedLock.unLockSafely(queryLockKey); + } + } else { + // 等待过程中,没有获取到锁,当前线程重新处理获取一遍值,一直拿不到可完蛋 + getLock(namespace, ++count); + } } return result; } @@ -50,6 +107,9 @@ public void evict(String namespace) { */ @PostConstruct public void reset() { + if (!isClear) { + return; + } String beClearKey = this.cacheName + "*"; if (!"*".equals(beClearKey)) { Set keys = redisTemplate.keys(beClearKey); @@ -66,8 +126,7 @@ public void reset() { */ private String getCacheKey(String namespace) { return CommonUtil.generateRedisKey(cacheName, - WebPluginUtils.traceTenantCode(), WebPluginUtils.traceEnvCode(), namespace); - + WebPluginUtils.traceTenantCode(), WebPluginUtils.traceEnvCode(), namespace); } /** @@ -85,7 +144,7 @@ protected T queryValue(String userAppKey, String envCode, String namespace) { /** * 新版本貌似用上面的方法替代了 - * + *

* TODO 具体实现 - 张天赐修改,为了编译通过 * * @param namespace - From 9d18de116170a7ee0a7852f9639462851546642c Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 14 Jul 2022 17:00:03 +0800 Subject: [PATCH 35/84] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E5=90=AF=E5=8A=A8,?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E7=9B=B8=E5=85=B3=E9=85=8D=E7=BD=AE=E4=B8=8D?= =?UTF-8?q?=E9=9C=80=E8=A6=81=E5=90=8C=E6=AD=A5=E5=88=B0zk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/biz/init/sync/ConfigSynchronizer.java | 46 +++++++++++-------- .../enums/config/ConfigServerKeyEnum.java | 12 +++-- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java index 03b9456fdf..4acdf85412 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java @@ -3,7 +3,9 @@ import java.util.List; import io.shulie.takin.web.biz.service.ApplicationService; +import io.shulie.takin.web.common.enums.config.ConfigServerKeyEnum; import io.shulie.takin.web.data.result.application.ApplicationDetailResult; +import io.shulie.takin.web.data.util.ConfigServerHelper; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -32,28 +34,34 @@ public void initSyncAgentConfig() { if (WebPluginUtils.checkUserPlugin()) { return; } - log.info("项目启动，重新同步信息去配置中心"); - List applications = applicationService.getAllApplications(); - if (CollectionUtils.isEmpty(applications)) { - return; + // 是否需要同步数据到配置中心 + if (!ConfigServerHelper.getBooleanValueByKey(ConfigServerKeyEnum.TAKIN_ENABLE_SYN_CONFIG)) { + log.warn("项目启动，应用相关配置不同步"); } else { - for (ApplicationDetailResult application : applications) { - configSyncService.syncGuard(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowDB(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncAllowList(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowJob(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowConsumer(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); + log.info("项目启动，重新同步信息去配置中心"); + List applications = applicationService.getAllApplications(); + if (CollectionUtils.isEmpty(applications)) { + return; + } else { + for (ApplicationDetailResult application : applications) { + configSyncService.syncGuard(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowDB(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncAllowList(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowJob(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowConsumer(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + } } } + log.info("项目启动，重新同步信息去配置中心"); configSyncService.syncClusterTestSwitch(WebPluginUtils.traceTenantCommonExt()); sleep(); configSyncService.syncAllowListSwitch(WebPluginUtils.traceTenantCommonExt()); diff --git a/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java b/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java index bec23eebba..3e9cd07d46 100644 --- a/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java +++ b/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java @@ -26,7 +26,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 是否执行添加linux用户的操作 */ TAKIN_FILE_OPS_SCRIPT_DEPLOY_USER_ENABLE("file.ops_script.deploy_user_enable", - "takin.file.ops_script.deploy_user_enable", NO), + "takin.file.ops_script.deploy_user_enable", NO), /** * 白名单长度校验 @@ -37,7 +37,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 是否开启白名单重名校验 */ TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK("whitelist.duplicate.name.check", "takin.white_list.duplicate.name.check", - YES), + YES), /** * 白名单文件 */ @@ -122,7 +122,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 压测引擎上传时间间隔 单位毫秒 */ TAKIN_PRESSURE_MACHINE_UPLOAD_INTERVAL_TIME("pressure.machine.upload.interval.time", - "takin.pressure.machine.upload.interval.time", NO), + "takin.pressure.machine.upload.interval.time", NO), /** * 脚本调试支持的 rpcType mq 下的 @@ -146,7 +146,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 等待时间 单位秒 */ TAKIN_PRADAR_SWITCH_PROCESSING_WAIT_TIME("pradar.switch.processing.wait.time", - "takin.pradar.switch.processing.wait.time", NO), + "takin.pradar.switch.processing.wait.time", NO), /** * 租户 id @@ -278,6 +278,10 @@ public enum ConfigServerKeyEnum implements AppConstants { */ TAKIN_TENANT_DEFAULT_PASSWORD("", "takin.tenant.default.password", YES), + /** + * 是否同步数据到zk + */ + TAKIN_ENABLE_SYN_CONFIG("", "takin.enable.syn.config", NO), ; /** From 0f80c7a9a0995c149305d6d04a94c030590218ce Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Thu, 14 Jul 2022 17:03:19 +0800 Subject: [PATCH 36/84] =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=A4=B1=E6=95=88,?= =?UTF-8?q?=E6=B5=81=E9=87=8F=E9=80=8F=E4=BC=A0=E5=88=B0mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../takin/web/app/ClearDsDataConfig.java | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java deleted file mode 100644 index 7cdd00b5eb..0000000000 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/ClearDsDataConfig.java +++ /dev/null @@ -1,28 +0,0 @@ -package io.shulie.takin.web.app; - -import io.shulie.takin.web.biz.cache.AbstractAgentConfigCache; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; -import org.springframework.stereotype.Component; - -import java.util.List; - -/** - * @Author: 南风 - * @Date: 2022/7/5 9:36 上午 - */ -@Component -@Slf4j -public class ClearDsDataConfig implements ApplicationRunner { - - @Autowired - private List cacheList; - - @Override - public void run(ApplicationArguments args) throws Exception { - //log.info("spring启动成功后重新清理缓存"); - //cacheList.forEach(AbstractAgentConfigCache::reset); - } -} From eb7e2b4a3fd723a38c5d74eadb52416e93a68d5a Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Fri, 15 Jul 2022 11:18:38 +0800 Subject: [PATCH 37/84] =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=A4=B1=E6=95=88,?= =?UTF-8?q?=E6=B5=81=E9=87=8F=E9=80=8F=E4=BC=A0=E5=88=B0mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/biz/cache/AbstractAgentConfigCache.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index ba6e945fad..c816721015 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -31,6 +31,10 @@ public abstract class AbstractAgentConfigCache implements AgentCacheSupport 50) { + if (count > 10) { log.warn("已超过递归次数,但还是未获取到值,namespace:", namespace); return null; } @@ -75,12 +79,12 @@ public T getLock(String namespace, int count) { if (result == null) { // 单独走一个查询的分布式key String queryLockKey = "t:data:query:" + cacheKey; - // 10ms去拿一次,操作很快会完成 - boolean isLock = distributedLock.tryLock(queryLockKey, 20L, 1000L, TimeUnit.MILLISECONDS); + // 多等待下再去获取值,读取数据也不是那么快 + boolean isLock = distributedLock.tryLock(queryLockKey, 100L, 1000L, TimeUnit.MILLISECONDS); if (isLock) { try { result = queryValue(namespace); - redisTemplate.opsForValue().set(cacheKey, result, 5, TimeUnit.MINUTES); + redisTemplate.opsForValue().set(cacheKey, result, app_config_expire, TimeUnit.MINUTES); } catch (Throwable e) { log.error("数据操作失败 " + ExceptionUtils.getStackTrace(e)); } finally { From 902d4add378f4dd2b5d6e7ca8331ab70c4966521 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Fri, 15 Jul 2022 11:33:07 +0800 Subject: [PATCH 38/84] =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=A4=B1=E6=95=88,?= =?UTF-8?q?=E6=B5=81=E9=87=8F=E9=80=8F=E4=BC=A0=E5=88=B0mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index c816721015..268f2505e5 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -70,7 +70,7 @@ public T get(String namespace) { */ public T getLock(String namespace, int count) { // 增加一个计数器,防止线程一直读取不到,递归退出,1s就退出，等待下次处理 - if (count > 10) { + if (count > 20) { log.warn("已超过递归次数,但还是未获取到值,namespace:", namespace); return null; } From a74f9b9b4ace0fea29c3b9f55624bb726c5cc181 Mon Sep 17 00:00:00 2001 From: zz052831 Date: Fri, 15 Jul 2022 17:09:55 +0800 Subject: [PATCH 39/84] =?UTF-8?q?=E6=89=93=E5=8C=85=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=97=B6=E5=88=86=E7=A7=92=E6=96=B9=E4=BE=BF=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- takin-web-app/pom.xml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/takin-web-app/pom.xml b/takin-web-app/pom.xml index fd243210e9..d9292433fd 100644 --- a/takin-web-app/pom.xml +++ b/takin-web-app/pom.xml @@ -388,7 +388,7 @@ - ${project.version} + src/main/resources @@ -422,7 +422,23 @@ - + + org.codehaus.mojo + buildnumber-maven-plugin + 1.4 + + yyMMddHHmmss + + + + + create-timestamp + + + + false + + ${project.version}-${timestamp} From fe5d0e426d328be8424e6523d0803dadf921a367 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Mon, 18 Jul 2022 11:52:54 +0800 Subject: [PATCH 40/84] merge --- .../service/impl/ApplicationServiceImpl.java | 95 ++++++++++--------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 713ed96204..e2ee6950e6 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -365,50 +365,57 @@ public Response> getApplicationList(ApplicationQueryRequest @Override public Long getAccessErrorNum() { - List results = applicationDAO.getDashboardAppData(); - // 通过amdb查询状态 - if (results == null || results.size() == 0) { - return 0L; - } - //取应用节点数信息 - List appNameList = results.stream().map(ApplicationDetailResult::getApplicationName).collect( - Collectors.toList()); - List applicationResultList = applicationDAO.getApplicationByName(appNameList); - if (CollectionUtil.isEmpty(applicationResultList)) { - return (long) results.size(); - } - Map> appResultMap = applicationResultList.stream() - .collect(Collectors.groupingBy(ApplicationResult::getAppName)); - //取应用节点版本信息 - ApplicationNodeQueryParam queryParam = new ApplicationNodeQueryParam(); - queryParam.setCurrent(0); - queryParam.setPageSize(99999); - queryParam.setApplicationNames(appNameList); - PagingList applicationNodes = applicationNodeDAO.pageNodes(queryParam); - if (CollectionUtil.isEmpty(applicationResultList)) { - return (long) results.size(); - } - - List applicationNodeResultList = applicationNodes.getList(); - Map> applicationNodeResultMap = applicationNodeResultList - .stream().collect(Collectors.groupingBy(ApplicationNodeResult::getAppName)); - return results.stream().filter(result -> { - List nodeResults = applicationNodeResultMap.get(result.getApplicationName()); - List appResults = appResultMap.get(result.getApplicationName()); - if (CollectionUtils.isEmpty(nodeResults) || CollectionUtils.isEmpty(appResults)) { - return true; - } - if (!appResults.get(0).getInstanceInfo().getInstanceOnlineAmount().equals(result.getNodeNum()) - || nodeResults.stream().map(ApplicationNodeResult::getAgentVersion).distinct().count() > 1) { - return true; - } - // 自身异常 - if (AppAccessStatusEnum.EXCEPTION.getCode().equals(result.getAccessStatus())) { - return true; - } - return false; - }).count(); - } + ApplicationQueryRequestV2 requestV2 = new ApplicationQueryRequestV2(); + requestV2.setAccessStatus(3); + return this.pageApplication(requestV2).getTotal(); + } + +// @Override +// public Long getAccessErrorNum() { +// List results = applicationDAO.getDashboardAppData(); +// // 通过amdb查询状态 +// if (results == null || results.size() == 0) { +// return 0L; +// } +// //取应用节点数信息 +// List appNameList = results.stream().map(ApplicationDetailResult::getApplicationName).collect( +// Collectors.toList()); +// List applicationResultList = applicationDAO.getApplicationByName(appNameList); +// if (CollectionUtil.isEmpty(applicationResultList)) { +// return (long) results.size(); +// } +// Map> appResultMap = applicationResultList.stream() +// .collect(Collectors.groupingBy(ApplicationResult::getAppName)); +// //取应用节点版本信息 +// ApplicationNodeQueryParam queryParam = new ApplicationNodeQueryParam(); +// queryParam.setCurrent(0); +// queryParam.setPageSize(99999); +// queryParam.setApplicationNames(appNameList); +// PagingList applicationNodes = applicationNodeDAO.pageNodes(queryParam); +// if (CollectionUtil.isEmpty(applicationResultList)) { +// return (long) results.size(); +// } +// +// List applicationNodeResultList = applicationNodes.getList(); +// Map> applicationNodeResultMap = applicationNodeResultList +// .stream().collect(Collectors.groupingBy(ApplicationNodeResult::getAppName)); +// return results.stream().filter(result -> { +// List nodeResults = applicationNodeResultMap.get(result.getApplicationName()); +// List appResults = appResultMap.get(result.getApplicationName()); +// if (CollectionUtils.isEmpty(nodeResults) || CollectionUtils.isEmpty(appResults)) { +// return true; +// } +// if (!appResults.get(0).getInstanceInfo().getInstanceOnlineAmount().equals(result.getNodeNum()) +// || nodeResults.stream().map(ApplicationNodeResult::getAgentVersion).distinct().count() > 1) { +// return true; +// } +// // 自身异常 +// if (AppAccessStatusEnum.EXCEPTION.getCode().equals(result.getAccessStatus())) { +// return true; +// } +// return false; +// }).count(); +// } @Override public List getApplicationListVo(ApplicationQueryRequest queryParam) { From 3600719e65752830ae6a36cde29a48d3f9699392 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Thu, 21 Jul 2022 20:46:21 +0800 Subject: [PATCH 41/84] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/biz/service/impl/ApplicationServiceImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index e2ee6950e6..39d4e8fdee 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -367,7 +367,7 @@ public Response> getApplicationList(ApplicationQueryRequest public Long getAccessErrorNum() { ApplicationQueryRequestV2 requestV2 = new ApplicationQueryRequestV2(); requestV2.setAccessStatus(3); - return this.pageApplication(requestV2).getTotal(); + return this.pageApplication(requestV2).getTotal(); } // @Override @@ -1326,8 +1326,15 @@ public PagingList pageApplication(ApplicationQueryReq List responseList = records.stream().map(result -> { ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); + String name = result.getApplicationName(); + Map map = applicationDAO.getStatus(name); + long n = (long) map.get("n"); + if (n != 0) { + result.setAccessStatus(3); + } return response; }).collect(Collectors.toList()); + return PagingList.of(responseList, applicationListResultPage.getTotal()); } From f1ecd43f286da8b59379647f0c08954ab0c6f1c3 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Thu, 21 Jul 2022 21:42:33 +0800 Subject: [PATCH 42/84] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E4=B8=8D=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ApplicationServiceImpl.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 39d4e8fdee..ad73d36596 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -713,6 +713,7 @@ public synchronized void syncApplicationAccessStatus() { // 正常的应用 Set normalApplicationIdSet = new HashSet<>(20); + Map errorInfo = Maps.newHashMap(); // 遍历比对 for (ApplicationListResult application : applicationList) { String applicationName = application.getApplicationName(); @@ -729,6 +730,8 @@ public synchronized void syncApplicationAccessStatus() { || !Objects.equals(amdbApplication.getInstanceInfo().getInstanceOnlineAmount(), nodeNum)) { // amdbApplicationMap 不存在, map.get 不存在, 或者节点数不一致 errorApplicationIdSet.add(applicationId); + errorInfo.put(applicationId, "节点数不一致"); + } else if (!amdbApplicationMap.isEmpty() && (amdbApplication = amdbApplicationMap.get(applicationName)) != null @@ -751,7 +754,7 @@ public synchronized void syncApplicationAccessStatus() { } - this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet); + this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet, errorInfo); } while (applicationNumber == pageSize); // 先执行一遍, 然后如果分页应用数量等于pageSize, 那么查询下一页 @@ -761,13 +764,18 @@ public synchronized void syncApplicationAccessStatus() { log.debug("定时同步应用状态完成!"); } - private void syncApplicationAccessStatus(List applicationList, Set errorApplicationIdSet) { + private void syncApplicationAccessStatus(List applicationList + , Set errorApplicationIdSet + , Map errorInfo) { if (CollectionUtils.isNotEmpty(applicationList)) { for (ApplicationListResult app : applicationList) { Map result = applicationDAO.getStatus(app.getApplicationName()); long n = (long) result.get("n"); if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = (String) result.get("e"); + if (io.shulie.takin.utils.string.StringUtil.isEmpty(e)) { + e = errorInfo.get(app.getApplicationId()); + } //不知道异常和Ip就别展示出来误导了 if (StringUtils.isBlank(e)) { String a = (String) result.get("a"); @@ -1326,15 +1334,8 @@ public PagingList pageApplication(ApplicationQueryReq List responseList = records.stream().map(result -> { ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); - String name = result.getApplicationName(); - Map map = applicationDAO.getStatus(name); - long n = (long) map.get("n"); - if (n != 0) { - result.setAccessStatus(3); - } return response; }).collect(Collectors.toList()); - return PagingList.of(responseList, applicationListResultPage.getTotal()); } From 9de5aeee8fde4fdec46b6955dcf54f949b5fc40c Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 10:10:37 +0800 Subject: [PATCH 43/84] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ApplicationServiceImpl.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index ad73d36596..102436ee2f 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -713,7 +713,6 @@ public synchronized void syncApplicationAccessStatus() { // 正常的应用 Set normalApplicationIdSet = new HashSet<>(20); - Map errorInfo = Maps.newHashMap(); // 遍历比对 for (ApplicationListResult application : applicationList) { String applicationName = application.getApplicationName(); @@ -730,8 +729,6 @@ public synchronized void syncApplicationAccessStatus() { || !Objects.equals(amdbApplication.getInstanceInfo().getInstanceOnlineAmount(), nodeNum)) { // amdbApplicationMap 不存在, map.get 不存在, 或者节点数不一致 errorApplicationIdSet.add(applicationId); - errorInfo.put(applicationId, "节点数不一致"); - } else if (!amdbApplicationMap.isEmpty() && (amdbApplication = amdbApplicationMap.get(applicationName)) != null @@ -754,7 +751,7 @@ public synchronized void syncApplicationAccessStatus() { } - this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet, errorInfo); + this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet); } while (applicationNumber == pageSize); // 先执行一遍, 然后如果分页应用数量等于pageSize, 那么查询下一页 @@ -764,24 +761,19 @@ public synchronized void syncApplicationAccessStatus() { log.debug("定时同步应用状态完成!"); } - private void syncApplicationAccessStatus(List applicationList - , Set errorApplicationIdSet - , Map errorInfo) { + private void syncApplicationAccessStatus(List applicationList, Set errorApplicationIdSet) { if (CollectionUtils.isNotEmpty(applicationList)) { for (ApplicationListResult app : applicationList) { Map result = applicationDAO.getStatus(app.getApplicationName()); long n = (long) result.get("n"); if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = (String) result.get("e"); - if (io.shulie.takin.utils.string.StringUtil.isEmpty(e)) { - e = errorInfo.get(app.getApplicationId()); - } - //不知道异常和Ip就别展示出来误导了 + if (StringUtils.isBlank(e)) { String a = (String) result.get("a"); - if (StringUtils.isEmpty(a)) { - continue; - } +// if (StringUtils.isEmpty(a)) { +// continue; +// } e = "探针接入异常"; if (StringUtils.isNotEmpty(a)) { e += "，agentId为" + a; From 1e7f31b428b5e255fb97ca7ad13b1f958b58e886 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 12:17:20 +0800 Subject: [PATCH 44/84] fix --- .../web/biz/service/impl/ApplicationServiceImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 102436ee2f..b717142637 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -771,9 +771,11 @@ private void syncApplicationAccessStatus(List application if (StringUtils.isBlank(e)) { String a = (String) result.get("a"); -// if (StringUtils.isEmpty(a)) { -// continue; -// } + if (StringUtils.isEmpty(a)) { + //只更新数据库状态为异常，不更新异常信息了，因为异常信息不知道为啥。 + applicationDAO.updateStatus(app.getApplicationId(), e); + continue; + } e = "探针接入异常"; if (StringUtils.isNotEmpty(a)) { e += "，agentId为" + a; From 2f010ee32a119bb3d2cda3f7e44c6059a3fd690a Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 12:35:16 +0800 Subject: [PATCH 45/84] fix --- .../impl/ApplicationErrorServiceImpl.java | 50 +++++++++---------- .../service/impl/ApplicationServiceImpl.java | 19 +++++++ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java index 42c3776b51..ed07bb067b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java @@ -66,15 +66,15 @@ public List list(ApplicationErrorQueryInput queryRequest // 应用节点相关错误信息 ApplicationErrorOutput nodeErrorResponse = - this.getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); + this.getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); if (nodeErrorResponse != null) { responseList.add(nodeErrorResponse); } //redisKey改造 String appUniqueKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - queryRequest.getApplicationId() + ApplicationServiceImpl.PRADARNODE_KEYSET); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + queryRequest.getApplicationId() + ApplicationServiceImpl.PRADARNODE_KEYSET); Set keys = redisTemplate.opsForSet().members(appUniqueKey); if (keys == null || keys.size() == 0) { return responseList; @@ -93,9 +93,9 @@ public List list(ApplicationErrorQueryInput queryRequest return this.processErrorList(responseList); } - private ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInput queryRequest) { + public ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInput queryRequest) { Response applicationMntResponse = applicationService.getApplicationInfoForError( - String.valueOf(queryRequest.getApplicationId())); + String.valueOf(queryRequest.getApplicationId())); ApplicationDetailResult tApplicationMnt = applicationMntResponse.getData(); if (Objects.isNull(tApplicationMnt)) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, "应用不存在"); @@ -104,11 +104,11 @@ private ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInpu } private void putNodeExceptionIfNeeded(List responseList, - ApplicationDetailResult tApplicationMnt) { + ApplicationDetailResult tApplicationMnt) { Integer totalNodeCount = tApplicationMnt.getNodeNum(); Integer onlineNodeCount = 0; List applicationResultList = applicationDAO.getApplicationByName( - Collections.singletonList(tApplicationMnt.getApplicationName())); + Collections.singletonList(tApplicationMnt.getApplicationName())); if (CollectionUtils.isEmpty(applicationResultList)) { log.error("AMDB中应用信息查询结果为空"); } else { @@ -120,16 +120,16 @@ private void putNodeExceptionIfNeeded(List responseList, } if (!totalNodeCount.equals(onlineNodeCount)) { responseList.add(new ApplicationErrorOutput() - .setExceptionId("-") - .setAgentIdList(Collections.singletonList("-")) - .setDescription("在线节点数 与 配置的节点总数 不一致") - .setTime(DateUtils.getNowDateStr()) - .setDetail("设置节点数：" + totalNodeCount + "，在线节点数：" + onlineNodeCount)); + .setExceptionId("-") + .setAgentIdList(Collections.singletonList("-")) + .setDescription("在线节点数 与 配置的节点总数 不一致") + .setTime(DateUtils.getNowDateStr()) + .setDetail("设置节点数：" + totalNodeCount + "，在线节点数：" + onlineNodeCount)); } } private void convertNodeUploadDataList(List responseList, - List nodeUploadDataDTOList) { + List nodeUploadDataDTOList) { nodeUploadDataDTOList.parallelStream().forEach(n -> { NodeUploadDataDTO nodeUploadDataDTO = JSONObject.parseObject(n, NodeUploadDataDTO.class); Map exceptionMap = nodeUploadDataDTO.getSwitchErrorMap(); @@ -177,8 +177,8 @@ public List getAppException(List appNames) { } //redisKey改造 String appUniqueKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceTenantCode(), - app.getAppId() + ApplicationServiceImpl.PRADAR_SEPERATE_FLAG); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceTenantCode(), + app.getAppId() + ApplicationServiceImpl.PRADAR_SEPERATE_FLAG); Set keys = redisTemplate.keys(appUniqueKey + "*"); if (keys != null) { for (String nodeKey : keys) { @@ -195,7 +195,7 @@ public List getAppException(List appNames) { if (message.contains("errorCode")) { try { ExceptionInfo exceptionInfo = JSONObject.parseObject(message, - ExceptionInfo.class); + ExceptionInfo.class); ApplicationExceptionOutput output = new ApplicationExceptionOutput(); output.setApplicationName(app.getAppName()); output.setAgentIds(Arrays.asList(nodeUploadDataDTO.getAgentId())); @@ -227,13 +227,13 @@ public List getAppException(List appNames) { * @param totalNodeCount 节点数量 * @return 节点错误 */ - private ApplicationErrorOutput getNodeErrorResponse(String applicationName, Integer totalNodeCount) { + public ApplicationErrorOutput getNodeErrorResponse(String applicationName, Integer totalNodeCount) { List applicationResultList = applicationDAO.getApplicationByName( - Collections.singletonList(applicationName)); + Collections.singletonList(applicationName)); ApplicationErrorOutput applicationErrorResponse = null; if (CollectionUtils.isEmpty(applicationResultList) - || !totalNodeCount.equals(applicationResultList.get(0).getInstanceInfo().getInstanceOnlineAmount())) { + || !totalNodeCount.equals(applicationResultList.get(0).getInstanceInfo().getInstanceOnlineAmount())) { applicationErrorResponse = new ApplicationErrorOutput(); applicationErrorResponse.setExceptionId("-"); applicationErrorResponse.setAgentIdList(Collections.singletonList("-")); @@ -259,14 +259,14 @@ private ApplicationErrorOutput getNodeErrorResponse(String applicationName, Inte private List processErrorList(List responseList) { // 按照时间倒序输出 List sortedList = responseList.parallelStream() - .filter(t -> t != null && CharSequenceUtil.isNotBlank(t.getTime())) - .sorted((a1, a2) -> a2.getTime().compareTo(a1.getTime())) - .collect(Collectors.toList()); + .filter(t -> t != null && CharSequenceUtil.isNotBlank(t.getTime())) + .sorted((a1, a2) -> a2.getTime().compareTo(a1.getTime())) + .collect(Collectors.toList()); List noTimeList = responseList.parallelStream() - // 无时间的 - .filter(response -> response != null && CharSequenceUtil.isBlank(response.getTime())) - .collect(Collectors.toList()); + // 无时间的 + .filter(response -> response != null && CharSequenceUtil.isBlank(response.getTime())) + .collect(Collectors.toList()); if (sortedList.isEmpty()) { return noTimeList; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index b717142637..bfc6e50632 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -43,6 +43,7 @@ import io.shulie.takin.web.biz.pojo.input.application.*; import io.shulie.takin.web.biz.pojo.input.whitelist.WhitelistImportFromExcelInput; import io.shulie.takin.web.biz.pojo.openapi.response.application.ApplicationListResponse; +import io.shulie.takin.web.biz.pojo.output.application.ApplicationErrorOutput; import io.shulie.takin.web.biz.pojo.request.activity.ActivityCreateRequest; import io.shulie.takin.web.biz.pojo.request.application.ApplicationListByUpgradeRequest; import io.shulie.takin.web.biz.pojo.request.application.ApplicationNodeOperateProbeRequest; @@ -55,7 +56,9 @@ import io.shulie.takin.web.biz.pojo.response.application.ShadowServerConfigurationResponse; import io.shulie.takin.web.biz.pojo.vo.application.ApplicationDsManageExportVO; import io.shulie.takin.web.biz.service.*; +import io.shulie.takin.web.biz.service.application.ApplicationErrorService; import io.shulie.takin.web.biz.service.application.ApplicationNodeService; +import io.shulie.takin.web.biz.service.application.impl.ApplicationErrorServiceImpl; import io.shulie.takin.web.biz.service.dsManage.DsService; import io.shulie.takin.web.biz.service.linkmanage.LinkGuardService; import io.shulie.takin.web.biz.service.linkmanage.WhiteListService; @@ -119,6 +122,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.annotation.PostConstruct; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.math.BigDecimal; @@ -1311,6 +1315,9 @@ public List getAllTenantApp(List commo return applicationDAO.getAllTenantApp(commonExtList); } + @Resource + ApplicationErrorServiceImpl applicationErrorService; + @Override public PagingList pageApplication(ApplicationQueryRequestV2 request) { QueryApplicationParam queryApplicationParam = BeanUtil.copyProperties(request, QueryApplicationParam.class); @@ -1328,6 +1335,18 @@ public PagingList pageApplication(ApplicationQueryReq List responseList = records.stream().map(result -> { ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); + + ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); + queryInput.setApplicationId(result.getApplicationId()); + ApplicationDetailResult tApplicationMnt = + applicationErrorService.ensureApplicationExist(queryInput); + ApplicationErrorOutput nodeErrorResponse = + applicationErrorService + .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); + if (nodeErrorResponse != null) { + response.setAccessStatus(3); + } + return response; }).collect(Collectors.toList()); return PagingList.of(responseList, applicationListResultPage.getTotal()); From efa4252ba4fc63eb4d371694dc2df256e13fff0b Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 13:14:16 +0800 Subject: [PATCH 46/84] =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ApplicationServiceImpl.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index bfc6e50632..805d93247b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -1336,16 +1336,16 @@ public PagingList pageApplication(ApplicationQueryReq ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); - ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); - queryInput.setApplicationId(result.getApplicationId()); - ApplicationDetailResult tApplicationMnt = - applicationErrorService.ensureApplicationExist(queryInput); - ApplicationErrorOutput nodeErrorResponse = - applicationErrorService - .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); - if (nodeErrorResponse != null) { - response.setAccessStatus(3); - } +// ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); +// queryInput.setApplicationId(result.getApplicationId()); +// ApplicationDetailResult tApplicationMnt = +// applicationErrorService.ensureApplicationExist(queryInput); +// ApplicationErrorOutput nodeErrorResponse = +// applicationErrorService +// .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); +// if (nodeErrorResponse != null) { +// response.setAccessStatus(3); +// } return response; }).collect(Collectors.toList()); From e3791567a06bdc5b6dc398c5b15c31c56db51185 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 13:33:59 +0800 Subject: [PATCH 47/84] fix --- .../biz/service/impl/ApplicationServiceImpl.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 805d93247b..80ad841c67 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -717,6 +717,7 @@ public synchronized void syncApplicationAccessStatus() { // 正常的应用 Set normalApplicationIdSet = new HashSet<>(20); + Map errorInfo = Maps.newHashMap(); // 遍历比对 for (ApplicationListResult application : applicationList) { String applicationName = application.getApplicationName(); @@ -733,6 +734,8 @@ public synchronized void syncApplicationAccessStatus() { || !Objects.equals(amdbApplication.getInstanceInfo().getInstanceOnlineAmount(), nodeNum)) { // amdbApplicationMap 不存在, map.get 不存在, 或者节点数不一致 errorApplicationIdSet.add(applicationId); + errorInfo.put(applicationId, "节点数不一致"); + } else if (!amdbApplicationMap.isEmpty() && (amdbApplication = amdbApplicationMap.get(applicationName)) != null @@ -755,7 +758,7 @@ public synchronized void syncApplicationAccessStatus() { } - this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet); + this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet, errorInfo); } while (applicationNumber == pageSize); // 先执行一遍, 然后如果分页应用数量等于pageSize, 那么查询下一页 @@ -765,7 +768,9 @@ public synchronized void syncApplicationAccessStatus() { log.debug("定时同步应用状态完成!"); } - private void syncApplicationAccessStatus(List applicationList, Set errorApplicationIdSet) { + private void syncApplicationAccessStatus(List applicationList + , Set errorApplicationIdSet + , Map errorInfo) { if (CollectionUtils.isNotEmpty(applicationList)) { for (ApplicationListResult app : applicationList) { Map result = applicationDAO.getStatus(app.getApplicationName()); @@ -776,8 +781,11 @@ private void syncApplicationAccessStatus(List application if (StringUtils.isBlank(e)) { String a = (String) result.get("a"); if (StringUtils.isEmpty(a)) { - //只更新数据库状态为异常，不更新异常信息了，因为异常信息不知道为啥。 - applicationDAO.updateStatus(app.getApplicationId(), e); + if (!io.shulie.takin.utils.string.StringUtil + .isEmpty(errorInfo.get(app.getApplicationId()))) { + //节点不一致 + applicationDAO.updateStatus(app.getApplicationId(), e); + } continue; } e = "探针接入异常"; From e3246c271ce2c385214e8298c6d71a34cc48a8a3 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 13:57:40 +0800 Subject: [PATCH 48/84] fix --- .../service/impl/ApplicationServiceImpl.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 80ad841c67..c98eada81b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -1344,16 +1344,16 @@ public PagingList pageApplication(ApplicationQueryReq ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); -// ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); -// queryInput.setApplicationId(result.getApplicationId()); -// ApplicationDetailResult tApplicationMnt = -// applicationErrorService.ensureApplicationExist(queryInput); -// ApplicationErrorOutput nodeErrorResponse = -// applicationErrorService -// .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); -// if (nodeErrorResponse != null) { -// response.setAccessStatus(3); -// } + ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); + queryInput.setApplicationId(result.getApplicationId()); + ApplicationDetailResult tApplicationMnt = + applicationErrorService.ensureApplicationExist(queryInput); + ApplicationErrorOutput nodeErrorResponse = + applicationErrorService + .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); + if (nodeErrorResponse != null) { + response.setAccessStatus(3); + } return response; }).collect(Collectors.toList()); From c30db08f3e415868484f7996998be1e2dbeee392 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 14:23:52 +0800 Subject: [PATCH 49/84] fix --- .../biz/service/impl/ApplicationServiceImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index c98eada81b..4fdbd22f69 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -499,6 +499,18 @@ public Response getApplicationInfo(String id) { if (tApplicationMnt == null) { return Response.success(new ApplicationVo()); } + //状态取amdb的，t_application_mnt中的不准 + ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); + queryInput.setApplicationId(tApplicationMnt.getApplicationId()); + + ApplicationErrorOutput nodeErrorResponse = + applicationErrorService + .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); + if (nodeErrorResponse != null) { + tApplicationMnt.setAccessStatus(3); + } else { + tApplicationMnt.setAccessStatus(0); + } // 取应用节点数信息 List applicationResultList = applicationDAO.getApplicationByName( @@ -1353,6 +1365,8 @@ public PagingList pageApplication(ApplicationQueryReq .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); if (nodeErrorResponse != null) { response.setAccessStatus(3); + } else { + response.setAccessStatus(0); } return response; From 9b4e9aa8e0324c8e672e8a2becd6114c15f24d24 Mon Sep 17 00:00:00 2001 From: Vernon2 <7615544+vernon2@user.noreply.gitee.com> Date: Fri, 22 Jul 2022 14:42:47 +0800 Subject: [PATCH 50/84] fix --- .../service/impl/ApplicationServiceImpl.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 4fdbd22f69..356c82b204 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -499,14 +499,13 @@ public Response getApplicationInfo(String id) { if (tApplicationMnt == null) { return Response.success(new ApplicationVo()); } - //状态取amdb的，t_application_mnt中的不准 + //判断是否有异常信息来获取状态，t_application_mnt中的不准 ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); queryInput.setApplicationId(tApplicationMnt.getApplicationId()); - ApplicationErrorOutput nodeErrorResponse = - applicationErrorService - .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); - if (nodeErrorResponse != null) { + List errors = + applicationErrorService.list(queryInput); + if (CollectionUtil.isNotEmpty(errors)) { tApplicationMnt.setAccessStatus(3); } else { tApplicationMnt.setAccessStatus(0); @@ -1356,14 +1355,12 @@ public PagingList pageApplication(ApplicationQueryReq ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); + ApplicationErrorQueryInput queryInput = new ApplicationErrorQueryInput(); queryInput.setApplicationId(result.getApplicationId()); - ApplicationDetailResult tApplicationMnt = - applicationErrorService.ensureApplicationExist(queryInput); - ApplicationErrorOutput nodeErrorResponse = - applicationErrorService - .getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); - if (nodeErrorResponse != null) { + List errors = + applicationErrorService.list(queryInput); + if (CollectionUtil.isNotEmpty(errors)) { response.setAccessStatus(3); } else { response.setAccessStatus(0); From 3652cad927c87fd935fb577852d59cc11022c3d3 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Mon, 25 Jul 2022 16:41:15 +0800 Subject: [PATCH 51/84] =?UTF-8?q?=E6=9E=84=E5=BB=BA=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E9=97=B4=E5=A4=A7=E4=B8=80=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shulie/takin/web/biz/cache/AbstractAgentConfigCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index 268f2505e5..32554213f6 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -70,8 +70,8 @@ public T get(String namespace) { */ public T getLock(String namespace, int count) { // 增加一个计数器,防止线程一直读取不到,递归退出,1s就退出，等待下次处理 - if (count > 20) { - log.warn("已超过递归次数,但还是未获取到值,namespace:", namespace); + if (count > 200) { + log.warn("已超过递归次数,但还是未获取到值,namespace:" + namespace); return null; } String cacheKey = getCacheKey(namespace); From 67170c6155a096b6f6726750a4b59035301f0d16 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 26 Jul 2022 11:07:55 +0800 Subject: [PATCH 52/84] =?UTF-8?q?=E6=9E=84=E5=BB=BA=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E9=97=B4=E5=A4=A7=E4=B8=80=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shulie/takin/web/biz/cache/AbstractAgentConfigCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index 32554213f6..bd56e64851 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -70,9 +70,9 @@ public T get(String namespace) { */ public T getLock(String namespace, int count) { // 增加一个计数器,防止线程一直读取不到,递归退出,1s就退出，等待下次处理 - if (count > 200) { + if (count > 100) { log.warn("已超过递归次数,但还是未获取到值,namespace:" + namespace); - return null; + throw new RuntimeException("数据获取超时,等待下次拉取!!!"); } String cacheKey = getCacheKey(namespace); T result = (T) redisTemplate.opsForValue().get(cacheKey); From b6964f1377ddedc336558fd2cefc18bcad7410c2 Mon Sep 17 00:00:00 2001 From: zhouyuan Date: Tue, 26 Jul 2022 15:01:40 +0800 Subject: [PATCH 53/84] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=B8=8B=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E4=B8=BA=E8=8E=B7=E5=8F=96=E5=88=B0,=E6=8A=9B?= =?UTF-8?q?=E5=87=BA=E5=BC=82=E5=B8=B8=EF=BC=8C=E9=81=BF=E5=85=8Dagent?= =?UTF-8?q?=E8=AE=A4=E4=B8=BA=E6=95=B0=E6=8D=AE=E5=A4=B1=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shulie/takin/web/biz/cache/AbstractAgentConfigCache.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index bd56e64851..ea14aff4af 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -71,8 +71,9 @@ public T get(String namespace) { public T getLock(String namespace, int count) { // 增加一个计数器,防止线程一直读取不到,递归退出,1s就退出，等待下次处理 if (count > 100) { - log.warn("已超过递归次数,但还是未获取到值,namespace:" + namespace); - throw new RuntimeException("数据获取超时,等待下次拉取!!!"); + log.warn("已超过递归次数,但还是未获取到值,namespace:{}", namespace); + // 返回失败,agent不会认为是失效的情况 + throw new RuntimeException("数据获取超时,等待下次拉取！"); } String cacheKey = getCacheKey(namespace); T result = (T) redisTemplate.opsForValue().get(cacheKey); From ea23f933ed1b62e270579417de150a6cf4873d7c Mon Sep 17 00:00:00 2001 From: nanfeng Date: Mon, 1 Aug 2022 16:18:22 +0800 Subject: [PATCH 54/84] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=90=8E,=E5=85=B3=E8=81=94=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=83=BD=E6=9F=A5=E5=87=BA=E6=9D=A5=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ApplicationPluginsConfigServiceImpl.java | 2 ++ .../impl/ShadowConsumerServiceImpl.java | 2 ++ .../impl/ApplicationApiServiceImpl.java | 2 ++ .../linkmanage/impl/LinkGuardServiceImpl.java | 3 ++- .../domain/query/ApplicationApiParam.java | 2 ++ .../data/dao/application/LinkGuardDAO.java | 2 ++ .../application/impl/LinkGuardDAOImpl.java | 21 +++++++++++++++++++ .../mappers/ApplicationApiManageMapper.xml | 3 +++ 8 files changed, 36 insertions(+), 1 deletion(-) diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java index 1b83c02100..c1b7f725b3 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java @@ -216,6 +216,8 @@ public List getListByParam(ApplicationPluginsConfigP if (Objects.isNull(param.getConfigKey())) { throw new TakinWebException(ExceptionCode.POD_NUM_EMPTY, "configKey为空"); } + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(param.getApplicationName()); + param.setApplicationId(application.getApplicationId()); List list = applicationPluginsConfigDAO.findList(param); if (list != null && !list.isEmpty()) { return CopyUtils.copyFieldsList(list, ApplicationPluginsConfigVO.class); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java index dcf22202f8..ee6e18eb8d 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java @@ -480,10 +480,12 @@ public void operateMqConsumers(ShadowConsumersOperateInput requests) { @Override public List agentSelect(String appName) { + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getDeleted, ShadowConsumerConstants.LIVED); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getStatus, ShadowConsumerConstants.ENABLE); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getApplicationName, appName); + lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getApplicationId, application.getApplicationId()); List entities = shadowMqConsumerMapper.selectList(lambdaQueryWrapper); if (CollectionUtils.isEmpty(entities)) { return Lists.newArrayList(); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java index 3fe2649083..c4f5dff6d3 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java @@ -179,8 +179,10 @@ public Response pullApi(String appName) { @Override public Map> pullApiV1(String appName) { + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); ApplicationApiParam apiParam = new ApplicationApiParam(); apiParam.setAppName(appName); + apiParam.setAppId(application.getApplicationId()); List all = applicationApiDAO.querySimpleWithTenant(apiParam); if (CollectionUtils.isEmpty(all)) { return null; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java index 3ba0e5712a..b615ec0e55 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java @@ -184,7 +184,8 @@ public Response> selectByExample(LinkGuardQueryParam param) { @Override public List agentSelect(String appName) { - List results = linkGuardDAO.selectByAppNameUnderCurrentUser(appName); + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); + List results = linkGuardDAO.selectByAppNameUnderCurrentUser(application.getApplicationId()); return results.stream().map(item -> { LinkGuardVo target = new LinkGuardVo(); BeanUtils.copyProperties(item, target); diff --git a/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java b/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java index eca42ba673..cfcc63c796 100644 --- a/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java +++ b/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java @@ -10,4 +10,6 @@ public class ApplicationApiParam extends BaseTenantBean { private String appName; + + private Long appId; } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java index 4017d503c9..c102ccd0a4 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java @@ -41,4 +41,6 @@ public interface LinkGuardDAO extends IService { */ void updateAppName(Long applicationId, String appName); + List selectByAppNameUnderCurrentUser(Long appId); + } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java index 93a5b2f06b..193a2ab0e2 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java @@ -93,4 +93,25 @@ public void updateAppName(Long applicationId, String appName) { this.update(wrapper); } + @Override + public List selectByAppNameUnderCurrentUser(Long appId) { + List results = new ArrayList<>(); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + if (WebPluginUtils.checkUserPlugin()) { + wrapper.eq(LinkGuardEntity::getTenantId, WebPluginUtils.traceTenantId()); + } + wrapper.eq(LinkGuardEntity::getApplicationId, appId); + List linkGuardEntities = this.getBaseMapper().selectList(wrapper); + if (CollectionUtils.isEmpty(linkGuardEntities)) { + return results; + } + return linkGuardEntities.stream().map(item -> { + LinkGuardResult target = new LinkGuardResult(); + BeanUtils.copyProperties(item, target); + // 启动 + target.setIsEnable(item.getIsEnable() == GuardEnableConstants.GUARD_ENABLE); + return target; + }).collect(Collectors.toList()); + } + } diff --git a/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml b/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml index d141111bf7..f0dc437ac8 100644 --- a/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml +++ b/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml @@ -170,6 +170,9 @@ and APPLICATION_NAME = #{appName} + + and APPLICATION_ID = #{appId} +