diff --git a/takin-web-amdb-accessor/pom.xml b/takin-web-amdb-accessor/pom.xml index 57f1bff9ca..f7e0da1e59 100644 --- a/takin-web-amdb-accessor/pom.xml +++ b/takin-web-amdb-accessor/pom.xml @@ -53,7 +53,7 @@ org.apache.commons commons-text - 1.8 + 1.10.0 compile @@ -73,6 +73,16 @@ io.shulie.takin takin-common-utils + + + commons-text + org.apache.commons + + + takin-common-exception-starter + io.shulie.takin + + org.mortbay.jetty @@ -93,6 +103,12 @@ io.shulie.takin takin-web-common + + + commons-text + org.apache.commons + + io.shulie.surge.data diff --git a/takin-web-app/pom.xml b/takin-web-app/pom.xml index fd243210e9..d9292433fd 100644 --- a/takin-web-app/pom.xml +++ b/takin-web-app/pom.xml @@ -388,7 +388,7 @@ - ${project.version} + src/main/resources @@ -422,7 +422,23 @@ - + + org.codehaus.mojo + buildnumber-maven-plugin + 1.4 + + yyMMddHHmmss + + + + + create-timestamp + + + + false + + ${project.version}-${timestamp} diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java index 02d683aa3e..af14db006e 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ElasticJobAutoConfig.java @@ -2,6 +2,13 @@ import com.dangdang.ddframe.job.event.JobEventConfiguration; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobOperateAPI; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobStatisticsAPI; +import com.dangdang.ddframe.job.lite.lifecycle.internal.operate.JobOperateAPIImpl; +import com.dangdang.ddframe.job.lite.lifecycle.internal.statistics.JobStatisticsAPIImpl; +import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter; +import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration; +import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter; import org.springframework.context.annotation.Bean; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; @@ -45,4 +52,24 @@ public SpringJobSchedulerFactory springJobSchedulerFactory(ElasticJobProperties public JobConfParser jobConfParser(SpringJobSchedulerFactory springJobSchedulerFactory) { return new JobConfParser(springJobSchedulerFactory); } + + // 初始化elasticJob页面操作类 + @Bean + public CoordinatorRegistryCenter zkCenter() { + ZookeeperConfiguration configuration = new + ZookeeperConfiguration(zkAddress, "takin-web-job-" + env); + CoordinatorRegistryCenter center = new ZookeeperRegistryCenter(configuration); + center.init(); + return center; + } + + @Bean + public JobStatisticsAPI jobStatisticsAPI(CoordinatorRegistryCenter zkCenter) { + return new JobStatisticsAPIImpl(zkCenter); + } + + @Bean + public JobOperateAPI jobOperateAPI(CoordinatorRegistryCenter zkCenter) { + return new JobOperateAPIImpl(zkCenter); + } } diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java new file mode 100644 index 0000000000..f18d1100c4 --- /dev/null +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/EsJobTaskListener.java @@ -0,0 +1,141 @@ +package io.shulie.takin.web.app.conf; + +import com.dangdang.ddframe.job.lite.lifecycle.api.JobOperateAPI; +import com.dangdang.ddframe.job.lite.lifecycle.api.JobStatisticsAPI; +import com.dangdang.ddframe.job.lite.lifecycle.domain.JobBriefInfo; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.shulie.takin.web.biz.service.DistributedLock; +import io.shulie.takin.web.common.util.JVMUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.File; +import java.io.FileOutputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.*; + +/** + * elasticjob 任务监听 + * + * @author zhouyuan + * @Date 7月5号 + */ +@Slf4j +@Component +public class EsJobTaskListener implements ApplicationListener { + // 需要监听的任务信息,按逗号分割 + @Value("${takin.listener.jobtask:}") + private String jobTask; + + @Value("${takin.listener.jobtask.initialDelay:600}") + private int initialDelay; + + @Value("${takin.listener.jobtask.period:30}") + private int period; + + // 是否打印jstack,如果状态不对的时候 + @Value("${takin.web.jstack.enable:false}") + private boolean jstackEnable; + + private static volatile long lastPrintTime = 0; + + private static final long TWO_MINUTES_MILLS = 2 * 60 * 1000; + + private static Semaphore guard = new Semaphore(1); + + private static final String USER_HOME = System.getProperty("user.home"); + + private static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd_HH:mm:ss"; + + @Resource + private JobOperateAPI jobOperateAPI; + + @Resource + private JobStatisticsAPI jobStatisticsAPI; + + @Resource + private DistributedLock distributedLock; + + /** + * @param event 事件源 + */ + @Override + public void onApplicationEvent(ApplicationStartedEvent event) { + try { + log.info("EsJob 任务监听开启 "); + if (StringUtils.isNotBlank(jobTask)) { + ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("JobTaskListener-%d").build()); + String esJobLock = "JobTaskListener_onApplicationEvent"; + // 检查job任务状态 + Runnable threadTask = new Runnable() { + @Override + public void run() { + try { + if (!distributedLock.tryLock(esJobLock, 1L, 1L, TimeUnit.MINUTES)) { + return; + } + // 获取任务状态 + String[] jobTaskArr = jobTask.split(","); + for (int i = 0; i < jobTaskArr.length; i++) { + // 获取任务状态 + JobBriefInfo jobBriefInfo = jobStatisticsAPI.getJobBriefInfo(jobTaskArr[i]); + if (jobBriefInfo == null) { + log.error("获取任务信息为空,{}", jobTaskArr[i]); + continue; + } + // 任务状态非正常 + if (jobBriefInfo.getStatus() == JobBriefInfo.JobStatus.SHARDING_FLAG) { + // 打印下jstack,看下当前线程在干啥 + if (jstackEnable) { + dumpJStack(); + } + // 点下触发,让他重试下 + log.info("当前任务状态为分片待调整,触发当前任务执行,{}", jobTaskArr[i]); + jobOperateAPI.trigger(com.google.common.base.Optional.of(jobTaskArr[i]), null); + } + } + } catch (Throwable e) { + log.error("任务处理失败,{}" + ExceptionUtils.getStackTrace(e)); + } finally { + distributedLock.unLock(esJobLock); + } + } + }; + // 提交任务,默认10分钟以后，每隔30秒执行一次 + service.scheduleAtFixedRate(threadTask, initialDelay, period, TimeUnit.SECONDS); + } + } catch (Throwable e) { + log.error("JobTaskListener is fail,{}" + ExceptionUtils.getStackTrace(e)); + } + } + + private void dumpJStack() { + long now = System.currentTimeMillis(); + //dump every 2 minutes + if (now - lastPrintTime < TWO_MINUTES_MILLS) { + return; + } + if (!guard.tryAcquire()) { + return; + } + String dumpPath = USER_HOME; + SimpleDateFormat sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT); + String dateStr = sdf.format(new Date()); + try (FileOutputStream jStackStream = new FileOutputStream( + new File(dumpPath, "Takin_web_JStack.log" + "." + dateStr))) { + JVMUtil.jstack(jStackStream); + } catch (Throwable t) { + log.error("dump jStack error", t); + } finally { + guard.release(); + } + lastPrintTime = System.currentTimeMillis(); + } +} diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java index d401c85bb2..8decb93524 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/ThreadPoolConfig.java @@ -32,6 +32,25 @@ @Configuration public class ThreadPoolConfig { + /** + * 用于定时任务 + * + * @return 线程池 + */ + @Bean(name = "syncAppStatusThreadPool") + public ThreadPoolExecutor syncAppStatusThreadPool() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("syncAppStatusThread-%d").build(); + return new ThreadPoolExecutor(10, 20, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "appRemoteApiFilterThreadPool") + public ThreadPoolExecutor appRemoteApiFilterThreadPool() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("appRemoteApiFilterThread-%d").build(); + return new ThreadPoolExecutor(10, 20, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + /** * 用于定时任务 * @@ -41,7 +60,7 @@ public class ThreadPoolConfig { public ThreadPoolExecutor jobThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-%d").build(); return new ThreadPoolExecutor(5, 10, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -53,7 +72,7 @@ public ThreadPoolExecutor jobThreadPool() { public ThreadPoolExecutor traceManageThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-trace-%d").build(); return new ThreadPoolExecutor(5, 10, 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -64,8 +83,29 @@ public ThreadPoolExecutor traceManageThreadPool() { @Bean(name = "collectDataThreadPool") public ThreadPoolExecutor collectDataThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-%d").build(); - return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_one") + public ThreadPoolExecutor collectDataThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-two-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_two") + public ThreadPoolExecutor collectDataThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-two-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "collectDataThreadPool_three") + public ThreadPoolExecutor collectDataThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-collect-data-three-%d").build(); + return new ThreadPoolExecutor(10, 50, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -77,7 +117,7 @@ public ThreadPoolExecutor collectDataThreadPool() { public ThreadPoolExecutor showdownVerifyThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-showdown-verify-%d").build(); return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -89,8 +129,29 @@ public ThreadPoolExecutor showdownVerifyThreadPool() { @Bean(name = "reportSummaryThreadPool") public ThreadPoolExecutor reportSummaryThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportSummaryThreadPool_one") + public ThreadPoolExecutor reportSummaryThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_one-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportSummaryThreadPool_two") + public ThreadPoolExecutor reportSummaryThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_two-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportSummaryThreadPool_three") + public ThreadPoolExecutor reportSummaryThreadPool_three() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-summary_three-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -102,8 +163,22 @@ public ThreadPoolExecutor reportSummaryThreadPool() { @Bean(name = "reportTpsThreadPool") public ThreadPoolExecutor reportTpsThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportTpsThreadPool_one") + public ThreadPoolExecutor reportTpsThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-one-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportTpsThreadPool_two") + public ThreadPoolExecutor reportTpsThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-tps-two-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -115,8 +190,22 @@ public ThreadPoolExecutor reportTpsThreadPool() { @Bean(name = "reportFinishThreadPool") public ThreadPoolExecutor reportFinishThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportFinishThreadPool_one") + public ThreadPoolExecutor reportFinishThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-one-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportFinishThreadPool_two") + public ThreadPoolExecutor reportFinishThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-finish-two-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -128,8 +217,22 @@ public ThreadPoolExecutor reportFinishThreadPool() { @Bean(name = "reportMachineThreadPool") public ThreadPoolExecutor reportMachineThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-%d").build(); - return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportMachineThreadPool_one") + public ThreadPoolExecutor reportMachineThreadPool_one() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-one-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Bean(name = "reportMachineThreadPool_two") + public ThreadPoolExecutor reportMachineThreadPool_two() { + ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("job-report-machine-two-%d").build(); + return new ThreadPoolExecutor(10, 20, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), nameThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); } /** @@ -141,14 +244,14 @@ public ThreadPoolExecutor reportMachineThreadPool() { public ThreadPoolExecutor fastDebug() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("fast-debug-%d").build(); return new ThreadPoolExecutor(5, 10, 20L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "modifyMonitorThreadPool") public ThreadPoolExecutor modifyMonitorExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("modify-monitor-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "schedulerPool") @@ -156,7 +259,7 @@ public TaskScheduler scheduler() { ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler(); taskScheduler.setPoolSize(5); ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("pradar-scheduler-thread-%d") - .daemon(true).build(); + .daemon(true).build(); taskScheduler.setThreadFactory(threadFactory); taskScheduler.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); return taskScheduler; @@ -166,29 +269,29 @@ public TaskScheduler scheduler() { public ThreadPoolExecutor runShellTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("script-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "loadDataThreadPool") public ThreadPoolExecutor loadDataTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("loaddata-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "agentDataThreadPool") public ThreadPoolExecutor agentDataTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agentdata-thread-%d").build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean(name = "backgroundMonitorThreadPool") public ThreadPoolExecutor backgroundMonitorThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("background-monitor-thread-%d") - .build(); + .build(); return new ThreadPoolExecutor(5, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Bean("asyncExecuteScriptThreadPool") @@ -216,7 +319,7 @@ public Executor myAsync() { public ThreadPoolExecutor runOPSShellTaskExecutor() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("ops-script-thread-%d").build(); return new ThreadPoolExecutor(1, 5, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Primary @@ -258,7 +361,7 @@ public HttpMessageConverters restFileDownloadSupport() { public ThreadPoolExecutor queryAsyncThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("query-async-thread-%d").build(); return new ThreadPoolExecutor(1, 10, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), nameThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.AbortPolicy()); } @Value("${poolConfig.e2e.coreSize: 20}") @@ -279,7 +382,7 @@ public ThreadPoolExecutor queryAsyncThreadPool() { public ThreadPoolExecutor e2eThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("e2e-job-%d").build(); return new ThreadPoolExecutor(e2eCoreSize, e2eMaxSize, 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(e2eQueueSize), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + new ArrayBlockingQueue<>(e2eQueueSize), nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -291,7 +394,7 @@ public ThreadPoolExecutor e2eThreadPool() { public ThreadPoolExecutor agentAggregationThreadPool() { ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agent-aggregation-%d").build(); return new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -304,7 +407,7 @@ public ThreadPoolExecutor middlewareResolverThreadPool() { final int coreSize = Runtime.getRuntime().availableProcessors(); ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("middleware-resolver-%d").build(); return new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + nameThreadFactory, new ThreadPoolExecutor.AbortPolicy()); } /** @@ -317,8 +420,8 @@ public ExecutorService agentHeartbeatThreadPool() { final int coreSize = Runtime.getRuntime().availableProcessors(); ThreadFactory nameThreadFactory = new ThreadFactoryBuilder().setNameFormat("agent-heartbeat-%d").build(); return TtlExecutors.getTtlExecutorService( - new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), - nameThreadFactory, new ThreadPoolExecutor.DiscardPolicy())); + new ThreadPoolExecutor(coreSize, coreSize * 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), + nameThreadFactory, new ThreadPoolExecutor.DiscardPolicy())); } /** diff --git a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/redis/RedisConfig.java b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/redis/RedisConfig.java index fa7a49f60a..e9aa08ffdb 100644 --- a/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/redis/RedisConfig.java +++ b/takin-web-app/src/main/java/io/shulie/takin/web/app/conf/redis/RedisConfig.java @@ -11,9 +11,11 @@ import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.json.JSONUtil; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator; import io.shulie.takin.web.common.constant.CacheConstants; import io.shulie.takin.web.ext.util.WebPluginUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -45,10 +47,10 @@ public class RedisConfig implements CacheConstants { @Bean(CACHE_KEY_GENERATOR_BY_TENANT_INFO) public KeyGenerator cacheKeyGeneratorByTenantInfo() { return (target, method, params) -> target.getClass().getName() - + method.getName() - + ":" - + Arrays.toString(DigestUtil.md5( - JSONUtil.toJsonStr(params) + WebPluginUtils.traceEnvCode() + WebPluginUtils.traceTenantId())); + + method.getName() + + ":" + + Arrays.toString(DigestUtil.md5( + JSONUtil.toJsonStr(params) + WebPluginUtils.traceEnvCode() + WebPluginUtils.traceTenantId())); } @Bean @@ -56,9 +58,9 @@ public RedisCacheManager cacheManager(RedisConnectionFactory redisConnectionFact // 自定义 redisCache 管理，指定key，带有过期时间，10分钟 // @Cacheable 使用 return new RedisCacheManager( - RedisCacheWriter.nonLockingRedisCacheWriter(redisConnectionFactory), - this.getRedisCacheConfigurationWithTtl(600), - CACHE_KEY_AGENT_CONFIG, CACHE_KEY_AGENT_APPLICATION_NODE, CACHE_KEY_ANNUAL_REPORT + RedisCacheWriter.nonLockingRedisCacheWriter(redisConnectionFactory), + this.getRedisCacheConfigurationWithTtl(600), + CACHE_KEY_AGENT_CONFIG, CACHE_KEY_AGENT_APPLICATION_NODE, CACHE_KEY_ANNUAL_REPORT ); } @@ -89,7 +91,7 @@ public RedisTemplate redisTemplate(RedisConnectionFactory factor RedisTemplate template = new RedisTemplate(); template.setConnectionFactory(factory); Jackson2JsonRedisSerializer jackson2JsonRedisSerializer - = getJackson2JsonRedisSerializer(); + = getJackson2JsonRedisSerializer(); StringRedisSerializer stringRedisSerializer = new StringRedisSerializer(); // key采用String的序列化方式 template.setKeySerializer(stringRedisSerializer); @@ -138,20 +140,17 @@ public RedisMessageListenerContainer container(RedisConnectionFactory factory) { */ private RedisCacheConfiguration getRedisCacheConfigurationWithTtl(Integer seconds) { RedisCacheConfiguration redisCacheConfiguration = RedisCacheConfiguration.defaultCacheConfig(); - redisCacheConfiguration = redisCacheConfiguration.serializeValuesWith( - RedisSerializationContext - .SerializationPair - .fromSerializer(this.getJackson2JsonRedisSerializer()) + redisCacheConfiguration = redisCacheConfiguration.serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(this.getJackson2JsonRedisSerializer()) ).entryTtl(Duration.ofSeconds(seconds)); return redisCacheConfiguration; } private Jackson2JsonRedisSerializer getJackson2JsonRedisSerializer() { Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>( - Object.class); + Object.class); ObjectMapper om = new ObjectMapper(); om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); - om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); + om.activateDefaultTyping(LaissezFaireSubTypeValidator.instance, ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); jackson2JsonRedisSerializer.setObjectMapper(om); return jackson2JsonRedisSerializer; diff --git a/takin-web-app/src/main/resources/application-local.properties b/takin-web-app/src/main/resources/application-local.properties index f95be3f9ad..a70e91a213 100644 --- a/takin-web-app/src/main/resources/application-local.properties +++ b/takin-web-app/src/main/resources/application-local.properties @@ -11,7 +11,7 @@ takin.config.zk.url=192.168.1.131:2181 spring.flyway.enabled=false # data.path=/Users/loseself/data/shulie -takin.config.zk.addr=192.168.1.204:2181 +takin.config.zk.addr=192.168.1.192:2181 spring.config.whiteListPath=/Users/loseself/data/whitelist # env=local diff --git a/takin-web-app/src/main/resources/application.properties b/takin-web-app/src/main/resources/application.properties index 750e806ac4..e295c12d44 100644 --- a/takin-web-app/src/main/resources/application.properties +++ b/takin-web-app/src/main/resources/application.properties @@ -141,4 +141,7 @@ takin.data.path=${data.path} takin.inner.pre=0 # 设置登录超时时间，默认24小时 -login.expire.time=86400 \ No newline at end of file +login.expire.time=86400 + +# 处理入口规则一次处理应用数 +takin.job.app.limit=10 \ No newline at end of file diff --git a/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java b/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java index f407b414c3..1f493bf79d 100644 --- a/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java +++ b/takin-web-app/src/test/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImplTest.java @@ -25,19 +25,26 @@ import io.shulie.takin.web.biz.pojo.request.datasource.DataSourceQueryRequest; import io.shulie.takin.web.biz.pojo.response.datasource.DatasourceListResponse; import io.shulie.takin.web.biz.service.DataSourceService; +import io.shulie.takin.web.biz.service.async.AsyncService; import io.shulie.takin.web.biz.service.linkmanage.AppRemoteCallService; import io.shulie.takin.web.common.enums.application.AppRemoteCallConfigEnum; +import io.shulie.takin.web.data.param.perfomanceanaly.PerformanceBaseDataParam; +import io.shulie.takin.web.data.param.perfomanceanaly.PerformanceThreadDataParam; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.security.core.parameters.P; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; +import java.util.ArrayList; +import java.util.List; + /** * 说明: * @@ -66,14 +73,37 @@ public class AppRemoteCallServiceImplTest { @Autowired DataSourceService dataSourceService; + @Autowired + AsyncService asyncService; + + @Test + public void zzTest(){ + PerformanceBaseDataParam param = new PerformanceBaseDataParam(); + param.setAgentId("11"); + param.setTimestamp(System.currentTimeMillis()); + List list = new ArrayList<>(); + for(int i=0;i<50;i++){ +// int i=9; + PerformanceThreadDataParam p1 = new PerformanceThreadDataParam(); + p1.setThreadStackLink(1L); + p1.setThreadId(i+"1"); + p1.setThreadName(i+"t1"); + p1.setGroupName(i+"g1"); + p1.setThreadCpuUsage(1D); + list.add(p1); + } + param.setThreadDataList(list); + asyncService.savePerformanceBaseData(param); + } + @Test public void batchConfigTest(){ - TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition); - AppRemoteCallConfigRequest request = new AppRemoteCallConfigRequest(); - request.setType(AppRemoteCallConfigEnum.OPEN_WHITELIST.getType().shortValue()); - request.setAppIds(Lists.newArrayList(6841250229157629952L,6846731729676275712L)); - appRemoteCallService.batchConfig(request); - dataSourceTransactionManager.commit(transactionStatus);//提交 +// TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition); +// AppRemoteCallConfigRequest request = new AppRemoteCallConfigRequest(); +// request.setType(AppRemoteCallConfigEnum.OPEN_WHITELIST.getType().shortValue()); +// request.setAppIds(Lists.newArrayList(6841250229157629952L,6846731729676275712L)); +// appRemoteCallService.batchConfig(request); +// dataSourceTransactionManager.commit(transactionStatus);//提交 } @Test @@ -92,15 +122,15 @@ public void agentTest(){ //System.out.println(completeSimulatorPath); } - @Test - public void like(){ - final DataSourceQueryRequest request = new DataSourceQueryRequest(); - request.setCurrent(0); - request.setPageSize(10); - //request.setDatasourceName("%"); - request.setJdbcUrl("_0"); - final PagingList list = dataSourceService.listDatasource( - request); - System.out.println(JSON.toJSONString(list)); - } +// @Test +// public void like(){ +// final DataSourceQueryRequest request = new DataSourceQueryRequest(); +// request.setCurrent(0); +// request.setPageSize(10); +// //request.setDatasourceName("%"); +// request.setJdbcUrl("_0"); +// final PagingList list = dataSourceService.listDatasource( +// request); +// System.out.println(JSON.toJSONString(list)); +// } } diff --git a/takin-web-biz-service/pom.xml b/takin-web-biz-service/pom.xml index 6eb832e68b..737a2179fc 100644 --- a/takin-web-biz-service/pom.xml +++ b/takin-web-biz-service/pom.xml @@ -162,6 +162,10 @@ commons-io commons-io + + annotation-indexer + org.jenkins-ci + diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java index e99d74ccb5..ea14aff4af 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/AbstractAgentConfigCache.java @@ -1,13 +1,20 @@ package io.shulie.takin.web.biz.cache; import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; +import io.shulie.takin.web.biz.service.DistributedLock; +import io.shulie.takin.web.biz.service.impl.RedissonDistributedLock; import io.shulie.takin.web.common.util.CommonUtil; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; /** @@ -20,6 +27,20 @@ public abstract class AbstractAgentConfigCache implements AgentCacheSupport 100) { + log.warn("已超过递归次数,但还是未获取到值,namespace:{}", namespace); + // 返回失败,agent不会认为是失效的情况 + throw new RuntimeException("数据获取超时,等待下次拉取！"); + } + String cacheKey = getCacheKey(namespace); + T result = (T) redisTemplate.opsForValue().get(cacheKey); if (result == null) { - result = queryValue(namespace); - redisTemplate.opsForValue().set(getCacheKey(namespace), result); + // 单独走一个查询的分布式key + String queryLockKey = "t:data:query:" + cacheKey; + // 多等待下再去获取值,读取数据也不是那么快 + boolean isLock = distributedLock.tryLock(queryLockKey, 100L, 1000L, TimeUnit.MILLISECONDS); + if (isLock) { + try { + result = queryValue(namespace); + redisTemplate.opsForValue().set(cacheKey, result, app_config_expire, TimeUnit.MINUTES); + } catch (Throwable e) { + log.error("数据操作失败 " + ExceptionUtils.getStackTrace(e)); + } finally { + distributedLock.unLockSafely(queryLockKey); + } + } else { + // 等待过程中,没有获取到锁,当前线程重新处理获取一遍值,一直拿不到可完蛋 + getLock(namespace, ++count); + } } return result; } @@ -48,7 +111,10 @@ public void evict(String namespace) { * 项目重启之后，缓存清空下 */ @PostConstruct - private void reset() { + public void reset() { + if (!isClear) { + return; + } String beClearKey = this.cacheName + "*"; if (!"*".equals(beClearKey)) { Set keys = redisTemplate.keys(beClearKey); @@ -65,8 +131,7 @@ private void reset() { */ private String getCacheKey(String namespace) { return CommonUtil.generateRedisKey(cacheName, - WebPluginUtils.traceTenantCode(), WebPluginUtils.traceEnvCode(), namespace); - + WebPluginUtils.traceTenantCode(), WebPluginUtils.traceEnvCode(), namespace); } /** @@ -84,7 +149,7 @@ protected T queryValue(String userAppKey, String envCode, String namespace) { /** * 新版本貌似用上面的方法替代了 - * + *

* TODO 具体实现 - 张天赐修改,为了编译通过 * * @param namespace - diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/agentimpl/ShadowKafkaClusterConfigAgentCache.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/agentimpl/ShadowKafkaClusterConfigAgentCache.java index 6d9bebecd2..70c708a0cc 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/agentimpl/ShadowKafkaClusterConfigAgentCache.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/cache/agentimpl/ShadowKafkaClusterConfigAgentCache.java @@ -17,11 +17,9 @@ @Component public class ShadowKafkaClusterConfigAgentCache extends AbstractAgentConfigCache> { - public static final String CACHE_NAME = "t:a:c:shadow:es"; - + public static final String CACHE_NAME = "t:a:c:shadow:kafka"; @Autowired private DsService dsService; - public ShadowKafkaClusterConfigAgentCache(@Autowired RedisTemplate redisTemplate) { super(CACHE_NAME, redisTemplate); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java index 98f7b768ab..30ba938ca5 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/common/AbstractSceneTask.java @@ -1,11 +1,8 @@ package io.shulie.takin.web.biz.common; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -63,7 +60,7 @@ protected int getAllowedTenantThreadMax() { return allowedTenantThreadMax; } allowedTenantThreadMax = ConfigServerHelper.getIntegerValueByKey( - ConfigServerKeyEnum.PER_TENANT_ALLOW_TASK_THREADS_MAX); + ConfigServerKeyEnum.PER_TENANT_ALLOW_TASK_THREADS_MAX); return allowedTenantThreadMax; } @@ -72,7 +69,7 @@ protected void cleanUnAvailableTasks(List taskDtoList) { if (CollectionUtils.isNotEmpty(taskDtoList)) { final LocalDateTime now = LocalDateTime.now(); taskDtoList.stream().filter(t -> t.getEndTime() != null && now.compareTo(t.getEndTime()) > 0).forEach( - t -> removeReportKey(t.getReportId())); + t -> removeReportKey(t.getReportId())); } } catch (Exception e) { log.error("清理过期任务时发生错误！", e); @@ -96,7 +93,7 @@ protected synchronized List runTask(List taskDtoList int allowedThreadMax = this.getAllowedTenantThreadMax(); //筛选出租户的任务 final Map> listMap = taskDtoList.stream().filter(t -> - t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() + t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() ).collect(Collectors.groupingBy(SceneTaskDto::getTenantId)); if (org.springframework.util.CollectionUtils.isEmpty(listMap)) { return taskAlreadyRun; @@ -111,7 +108,7 @@ protected synchronized List runTask(List taskDtoList * 取最值。当前租户的任务数和允许的最大线程数 */ AtomicInteger allowRunningThreads = new AtomicInteger( - Math.min(allowedThreadMax, tenantTasks.size())); + Math.min(allowedThreadMax, tenantTasks.size())); /** * 已经运行的任务数 @@ -128,7 +125,7 @@ protected synchronized List runTask(List taskDtoList * allow running threads calculated by capacity */ int permitsThreads = Math.min(allowedThreadMax - oldRunningThreads.get(), - allowRunningThreads.get()); + allowRunningThreads.get()); // add new threads to capacity oldRunningThreads.addAndGet(permitsThreads); // adjust allow current running threads @@ -144,4 +141,29 @@ protected synchronized List runTask(List taskDtoList return taskAlreadyRun; } + + /** + * @param taskDtoList + * @param shardingContext + */ + protected void runTask_ext(List taskDtoList, ShardingContext shardingContext) { + //筛选出租户的任务 + final Map> listMap = + taskDtoList.stream().filter(t -> t.getReportId() % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem() + ).collect(Collectors.groupingBy(SceneTaskDto::getTenantId)); + if (listMap.isEmpty()) { + return; + } + for (Entry> listEntry : listMap.entrySet()) { + final List tenantTasks = listEntry.getValue(); + if (CollectionUtils.isEmpty(tenantTasks)) { + continue; + } + for (int i = 0; i < tenantTasks.size(); i++) { + final SceneTaskDto task = tenantTasks.get(i); + this.runTaskInTenantIfNecessary(task, task.getReportId()); + } + } + } + } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java index 03b9456fdf..4acdf85412 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/init/sync/ConfigSynchronizer.java @@ -3,7 +3,9 @@ import java.util.List; import io.shulie.takin.web.biz.service.ApplicationService; +import io.shulie.takin.web.common.enums.config.ConfigServerKeyEnum; import io.shulie.takin.web.data.result.application.ApplicationDetailResult; +import io.shulie.takin.web.data.util.ConfigServerHelper; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -32,28 +34,34 @@ public void initSyncAgentConfig() { if (WebPluginUtils.checkUserPlugin()) { return; } - log.info("项目启动，重新同步信息去配置中心"); - List applications = applicationService.getAllApplications(); - if (CollectionUtils.isEmpty(applications)) { - return; + // 是否需要同步数据到配置中心 + if (!ConfigServerHelper.getBooleanValueByKey(ConfigServerKeyEnum.TAKIN_ENABLE_SYN_CONFIG)) { + log.warn("项目启动，应用相关配置不同步"); } else { - for (ApplicationDetailResult application : applications) { - configSyncService.syncGuard(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowDB(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncAllowList(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowJob(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); - sleep(); - configSyncService.syncShadowConsumer(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), - application.getApplicationName()); + log.info("项目启动，重新同步信息去配置中心"); + List applications = applicationService.getAllApplications(); + if (CollectionUtils.isEmpty(applications)) { + return; + } else { + for (ApplicationDetailResult application : applications) { + configSyncService.syncGuard(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowDB(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncAllowList(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowJob(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + sleep(); + configSyncService.syncShadowConsumer(WebPluginUtils.traceTenantCommonExt(), application.getApplicationId(), + application.getApplicationName()); + } } } + log.info("项目启动，重新同步信息去配置中心"); configSyncService.syncClusterTestSwitch(WebPluginUtils.traceTenantCommonExt()); sleep(); configSyncService.syncAllowListSwitch(WebPluginUtils.traceTenantCommonExt()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppAccessStatusJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppAccessStatusJob.java index cd7f1d6349..8e08b912a9 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppAccessStatusJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppAccessStatusJob.java @@ -28,17 +28,17 @@ */ @Component @ElasticSchedulerJob(jobName = "appAccessStatusJob", cron = "0/10 * * * * ?", description = "同步大数据应用状态", - // 时效转移 - misfire = true, - // 重新执行 - failover = true) + // 时效转移 + misfire = true, + // 重新执行 + failover = true) public class AppAccessStatusJob implements SimpleJob { @Autowired private ApplicationService applicationService; @Resource - @Qualifier("jobThreadPool") - private ThreadPoolExecutor jobThreadPool; + @Qualifier("syncAppStatusThreadPool") + private ThreadPoolExecutor syncAppStatusThreadPool; @Autowired private DistributedLock distributedLock; @@ -53,26 +53,26 @@ public void execute(ShardingContext shardingContext) { List tenantInfoExts = WebPluginUtils.getTenantInfoList(); for (TenantInfoExt ext : tenantInfoExts) { - if(CollectionUtils.isEmpty(ext.getEnvs())) { + if (CollectionUtils.isEmpty(ext.getEnvs())) { continue; } // 根据环境 分线程 for (TenantEnv e : ext.getEnvs()) { // 开始数据层分片 // 分布式锁 - String lockKey = JobRedisUtils.getJobRedis(ext.getTenantId(),e.getEnvCode(),shardingContext.getJobName()); + String lockKey = JobRedisUtils.getJobRedis(ext.getTenantId(), e.getEnvCode(), shardingContext.getJobName()); if (distributedLock.checkLock(lockKey)) { continue; } - jobThreadPool.execute(() -> { - boolean tryLock = distributedLock.tryLock(lockKey, 1L, 1L, TimeUnit.MINUTES); - if(!tryLock) { + syncAppStatusThreadPool.execute(() -> { + boolean tryLock = distributedLock.tryLock(lockKey, 0L, 1L, TimeUnit.MINUTES); + if (!tryLock) { return; } try { WebPluginUtils.setTraceTenantContext( - new TenantCommonExt(ext.getTenantId(),ext.getTenantAppKey(),e.getEnvCode(), - ext.getTenantCode(), ContextSourceEnum.JOB.getCode())); + new TenantCommonExt(ext.getTenantId(), ext.getTenantAppKey(), e.getEnvCode(), + ext.getTenantCode(), ContextSourceEnum.JOB.getCode())); applicationService.syncApplicationAccessStatus(); WebPluginUtils.removeTraceContext(); } finally { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppRemoteApiFilterJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppRemoteApiFilterJob.java index 8abeca6b4c..b468ab0ab9 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppRemoteApiFilterJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/AppRemoteApiFilterJob.java @@ -50,8 +50,8 @@ public class AppRemoteApiFilterJob implements SimpleJob { @Resource private ApplicationApiService apiService; @Resource - @Qualifier("jobThreadPool") - private ThreadPoolExecutor jobThreadPool; + @Qualifier("appRemoteApiFilterThreadPool") + private ThreadPoolExecutor appRemoteApiFilterThreadPool; @Resource private DistributedLock distributedLock; @@ -64,7 +64,7 @@ public void execute(ShardingContext shardingContext) { } else { List tenantInfoExtList = WebPluginUtils.getTenantInfoList(); for (TenantInfoExt ext : tenantInfoExtList) { - if(CollectionUtils.isEmpty(ext.getEnvs())) { + if (CollectionUtils.isEmpty(ext.getEnvs())) { continue; } // 根据环境 分线程 @@ -74,15 +74,15 @@ public void execute(ShardingContext shardingContext) { if (distributedLock.checkLock(lockKey)) { continue; } - jobThreadPool.execute(() -> { - boolean tryLock = distributedLock.tryLock(lockKey, 1L, 1L, TimeUnit.MINUTES); + appRemoteApiFilterThreadPool.execute(() -> { + boolean tryLock = distributedLock.tryLock(lockKey, 0L, 1L, TimeUnit.MINUTES); if (!tryLock) { return; } try { WebPluginUtils.setTraceTenantContext( - new TenantCommonExt(ext.getTenantId(), ext.getTenantAppKey(), e.getEnvCode(), - ext.getTenantCode(), ContextSourceEnum.JOB.getCode())); + new TenantCommonExt(ext.getTenantId(), ext.getTenantAppKey(), e.getEnvCode(), + ext.getTenantCode(), ContextSourceEnum.JOB.getCode())); this.appRemoteApiFilter(); WebPluginUtils.removeTraceContext(); } finally { @@ -121,7 +121,7 @@ private void appRemoteApiFilter() { } delList.addAll(appRemoteCallFilterList); // 唯一 - filterMap.put(apiManage.getApplicationId()+"##" +apiManage.getApi(), appRemoteCallFilterList); + filterMap.put(apiManage.getApplicationId() + "##" + apiManage.getApi(), appRemoteCallFilterList); }); }); @@ -132,7 +132,7 @@ private void appRemoteApiFilter() { List save = Lists.newArrayList(); filterMap.forEach((k, v) -> { String[] temp = k.split("##"); - if(temp.length != 2) { + if (temp.length != 2) { return; } String interfaceName = temp[1]; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java index f3f300a47a..61745d0892 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcApplicationSummaryJob.java @@ -2,94 +2,91 @@ import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import com.dangdang.ddframe.job.api.ShardingContext; import com.dangdang.ddframe.job.api.simple.SimpleJob; import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; /** * @author 无涯 * @date 2021/7/13 23:10 */ -@Component -@ElasticSchedulerJob(jobName = "calcApplicationSummaryJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - isSharding = true, - //shardingItemParameters = "0=0,1=1,2=2", - cron = "*/10 * * * * ?", - description = "汇总应用 机器数 风险机器数") +//@Component +//@ElasticSchedulerJob(jobName = "calcApplicationSummaryJob", +// // 分片序列号和参数用等号分隔 不需要参数可以不加 +// isSharding = true, +// //shardingItemParameters = "0=0,1=1,2=2", +// cron = "*/10 * * * * ?", +// description = "汇总应用 机器数 风险机器数") @Slf4j public class CalcApplicationSummaryJob extends AbstractSceneTask implements SimpleJob { @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportSummaryThreadPool") - private ThreadPoolExecutor reportThreadPool; - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.CalcApplicationSummaryJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.calcApplicationSummary(reportId); - } catch (Throwable e) { - log.error( + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current not running pressure task!!!"); + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportSummaryThreadPool().execute(() -> { + try { + reportTaskService.calcApplicationSummary(reportId); + } catch (Throwable e) { + log.error( "execute CalcApplicationSummaryJob occured error. reportId= {},errorMsg={}", reportId, e.getMessage(), e); - } finally { - runningTasks.remove(reportId); - } - }); - } + } finally { + runningTasks.remove(reportId); + } + }); } } - } else { - this.runTask(taskDtoList,shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } - log.debug("calcApplicationSummaryJob 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getReportSummaryThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.calcApplicationSummary(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java index f02e062d8c..3c93608969 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/CalcTpsTargetJob.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; @@ -11,24 +10,25 @@ import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** * @author 无涯 * @date 2021/7/13 23:10 */ -@Component -@ElasticSchedulerJob(jobName = "calcTpsTargetJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - //shardingItemParameters = "0=0,1=1,2=2", - isSharding = true, - cron = "*/10 * * * * ?", - description = "获取tps指标图") +//@Component +//@ElasticSchedulerJob(jobName = "calcTpsTargetJob", +// // 分片序列号和参数用等号分隔 不需要参数可以不加 +// //shardingItemParameters = "0=0,1=1,2=2", +// isSharding = true, +// cron = "*/10 * * * * ?", +// description = "获取tps指标图") @Slf4j public class CalcTpsTargetJob extends AbstractSceneTask implements SimpleJob { @@ -36,55 +36,58 @@ public class CalcTpsTargetJob extends AbstractSceneTask implements SimpleJob { private ReportTaskService reportTaskService; @Autowired - @Qualifier("reportTpsThreadPool") - private ThreadPoolExecutor reportThreadPool; + private ThreadPoolUtil threadPoolUtil; private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.CalcTpsTargetJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } - if (openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.calcTpsTarget(reportId); - } catch (Throwable e) { - log.error("execute CalcTpsTargetJob occured error. reportId={}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - - }); - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current not running pressure task!!!"); + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportTpsThreadPool().execute(() -> { + try { + reportTaskService.calcTpsTarget(reportId); + } catch (Throwable e) { + log.error("execute CalcTpsTargetJob occured error. reportId={}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } } - } else { - this.runTask(taskDtoList,shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } log.debug("calcTpsTargetJob 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + threadPoolUtil.getReportTpsThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.calcTpsTarget(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java index 141e75bcea..23237f6ee4 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/FinishReportJob.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; @@ -11,11 +10,12 @@ import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** @@ -24,69 +24,70 @@ */ @Component @ElasticSchedulerJob(jobName = "finishReportJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - //shardingItemParameters = "0=0,1=1,2=2", - isSharding = true, - cron = "*/10 * * * * ?", - description = "压测报告状态，汇总报告") + // 分片序列号和参数用等号分隔 不需要参数可以不加 + //shardingItemParameters = "0=0,1=1,2=2", + isSharding = true, + cron = "*/10 * * * * ?", + description = "压测报告状态，汇总报告") @Slf4j public class FinishReportJob extends AbstractSceneTask implements SimpleJob { @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportFinishThreadPool") - private ThreadPoolExecutor reportThreadPool; - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.FinishReportJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); //任务开始 - while (true){ - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } - if(openVersion) { - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - // 私有化 + 开源 根据 报告id进行分片 - // 开始数据层分片 - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.finishReport(reportId,taskDto); - } catch (Throwable e) { - log.error("execute FinishReportJob occured error. reportId={}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - }); - } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current not running pressure task!!!"); + return; + } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + // 私有化 + 开源 根据 报告id进行分片 + // 开始数据层分片 + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getReportFinishThreadPool().execute(() -> { + try { + reportTaskService.finishReport(reportId, taskDto); + } catch (Throwable e) { + log.error("execute FinishReportJob occured error. reportId={}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } } - this.cleanUnAvailableTasks(taskDtoList); - }else { - final List taskAlreadyRun = this.runTask(taskDtoList, shardingContext); - this.cleanUnAvailableTasks(taskAlreadyRun); } + this.cleanUnAvailableTasks(taskDtoList); + } else { + this.runTask_ext(taskDtoList, shardingContext); + this.cleanUnAvailableTasks(taskDtoList); } log.debug("finishReport 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override - protected void runTaskInTenantIfNecessary( SceneTaskDto tenantTask, Long reportId) { + protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getReportFinishThreadPool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.finishReport(reportId, tenantTask); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java index 9f4fe04298..ea50b65108 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/job/SyncMachineDataJob.java @@ -3,90 +3,92 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import com.dangdang.ddframe.job.api.ShardingContext; import com.dangdang.ddframe.job.api.simple.SimpleJob; +import com.google.common.collect.Maps; import io.shulie.takin.job.annotation.ElasticSchedulerJob; import io.shulie.takin.web.biz.common.AbstractSceneTask; import io.shulie.takin.web.biz.service.report.ReportTaskService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** * @author 无涯 * @date 2021/7/13 23:10 */ -@Component -@ElasticSchedulerJob(jobName = "syncMachineDataJob", - // 分片序列号和参数用等号分隔 不需要参数可以不加 - isSharding = true, - //shardingItemParameters = "0=0,1=1,2=2", - cron = "*/10 * * * * ?", - description = "同步应用基础信息") +//@Component +//@ElasticSchedulerJob(jobName = "syncMachineDataJob", +// // 分片序列号和参数用等号分隔 不需要参数可以不加 +// isSharding = true, +// //shardingItemParameters = "0=0,1=1,2=2", +// cron = "*/10 * * * * ?", +// description = "同步应用基础信息") @Slf4j public class SyncMachineDataJob extends AbstractSceneTask implements SimpleJob { @Autowired private ReportTaskService reportTaskService; - @Autowired - @Qualifier("reportMachineThreadPool") - private ThreadPoolExecutor reportThreadPool; - - private static Map runningTasks = new ConcurrentHashMap<>(); private static AtomicInteger EMPTY = new AtomicInteger(); + private Map syncMacheineMap = Maps.newHashMap(); + @Override public void execute(ShardingContext shardingContext) { + try { + this.execute_ext(shardingContext); + } catch (Throwable e) { + // 捕捉全部异常,防止任务异常，导致esjob有问题 + log.error("io.shulie.takin.web.biz.job.SyncMachineDataJob#execute error" + ExceptionUtils.getStackTrace(e)); + } + } + + public void execute_ext(ShardingContext shardingContext) { long start = System.currentTimeMillis(); final Boolean openVersion = WebPluginUtils.isOpenVersion(); - while (true) { - List taskDtoList = getTaskFromRedis(); - if (taskDtoList == null) { break; } + List taskDtoList = getTaskFromRedis(); + if (taskDtoList == null) { + log.warn("current not running pressure task!!!"); + return; + } - if (openVersion){ - for (SceneTaskDto taskDto : taskDtoList) { - Long reportId = taskDto.getReportId(); - if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { - Object task = runningTasks.putIfAbsent(reportId, EMPTY); - if (task == null) { - reportThreadPool.execute(() -> { - try { - reportTaskService.syncMachineData(reportId); - } catch (Throwable e) { - log.error("execute SyncMachineDataJob occured error. reportId= {}", reportId, e); - } finally { - runningTasks.remove(reportId); - } - }); - } + if (openVersion) { + for (SceneTaskDto taskDto : taskDtoList) { + Long reportId = taskDto.getReportId(); + if (reportId % shardingContext.getShardingTotalCount() == shardingContext.getShardingItem()) { + Object task = runningTasks.putIfAbsent(reportId, EMPTY); + if (task == null) { + ThreadPoolUtil.getSyncMachinePool().execute(() -> { + try { + reportTaskService.syncMachineData(reportId); + } catch (Throwable e) { + log.error("execute SyncMachineDataJob occured error. reportId= {}", reportId, e); + } finally { + runningTasks.remove(reportId); + } + }); } - } - }else { - this.runTask(taskDtoList,shardingContext); } + } else { + this.runTask_ext(taskDtoList, shardingContext); } - log.debug("syncMachineData 执行时间:{}", System.currentTimeMillis() - start); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } } @Override protected void runTaskInTenantIfNecessary(SceneTaskDto tenantTask, Long reportId) { //将任务放入线程池 - reportThreadPool.execute(() -> { + ThreadPoolUtil.getSyncMachinePool().execute(() -> { try { WebPluginUtils.setTraceTenantContext(tenantTask); reportTaskService.syncMachineData(tenantTask.getReportId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java index 84db709517..4cdde8e008 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/mq/consumer/impl/middleware/AgentPushMiddlewareAndCompareConsumer.java @@ -18,6 +18,7 @@ import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.connection.Message; import org.springframework.data.redis.connection.MessageListener; @@ -39,6 +40,10 @@ public class AgentPushMiddlewareAndCompareConsumer implements MessageListener { @Autowired private ApplicationMiddlewareDAO applicationMiddlewareDAO; + //默认不处理中间件信息 + @Value("${takin.enable.middlewareFlag:false}") + private boolean middlewareFlag; + @Autowired @Lazy private ApplicationMiddlewareService applicationMiddlewareService; @@ -46,6 +51,9 @@ public class AgentPushMiddlewareAndCompareConsumer implements MessageListener { @Transactional(rollbackFor = Throwable.class) @Override public void onMessage(Message message, byte[] pattern) { + if (!middlewareFlag) { + return; + } String messageBody = new String(message.getBody()); if (StringUtils.isEmpty(messageBody)) { return; @@ -53,7 +61,7 @@ public void onMessage(Message message, byte[] pattern) { messageBody = messageBody.substring(1, messageBody.length() - 1).replace("\\", ""); MqApplicationMiddlewareCompareDTO mqApplicationMiddlewareCompareDTO = JsonUtil.json2Bean(messageBody, - MqApplicationMiddlewareCompareDTO.class); + MqApplicationMiddlewareCompareDTO.class); if (mqApplicationMiddlewareCompareDTO == null) { return; } @@ -69,15 +77,15 @@ public void onMessage(Message message, byte[] pattern) { try { TenantCommonExt tenantCommonExt = new TenantCommonExt(mqApplicationMiddlewareCompareDTO.getTenantId(), - null, mqApplicationMiddlewareCompareDTO.getEnvCode(), - null, ContextSourceEnum.JOB.getCode()); + null, mqApplicationMiddlewareCompareDTO.getEnvCode(), + null, ContextSourceEnum.JOB.getCode()); WebPluginUtils.setTraceTenantContext(tenantCommonExt); // 根据 applicationId 查询应用中间件 log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件查询"); PageUtils.clearPageHelper(); List applicationMiddlewareList = - applicationMiddlewareDAO.listByApplicationId(applicationId); + applicationMiddlewareDAO.listByApplicationId(applicationId); if (applicationMiddlewareList.isEmpty()) { return; } @@ -85,7 +93,7 @@ public void onMessage(Message message, byte[] pattern) { // 比对 log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件比对"); List updateParamList = - applicationMiddlewareService.doCompare(applicationMiddlewareList); + applicationMiddlewareService.doCompare(applicationMiddlewareList); log.info("应用中间件上报 --> 异步消息处理 --> 应用中间件更新"); applicationMiddlewareDAO.updateBatchById(updateParamList); @@ -103,7 +111,7 @@ public void onMessage(Message message, byte[] pattern) { * @return 相应数据 */ private MqApplicationMiddlewareCompareDTO getMqApplicationMiddlewareCompareDTO( - Message message) { + Message message) { String messageBody = new String(message.getBody()); if (StringUtils.isEmpty(messageBody)) { return null; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java index ef0cb9b7ad..1ff7578fef 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/ConfCenterService.java @@ -99,6 +99,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -150,6 +151,10 @@ public class ConfCenterService extends CommonService { public static final String APPLICATION_CACHE_PREFIX = "application:cache"; + // 是否默认初始化白名单 + @Value("${takin.enable.initWhiteList:false}") + private boolean initWhiteList; + @PostConstruct public void init() { number = ConfigServerHelper.getWrapperIntegerValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_NUMBER_LIMIT); @@ -244,6 +249,10 @@ private void addApplicationToDataBuild(ApplicationCreateParam tApplicationMnt) { @PostConstruct public void initWhiteList() { + if (!initWhiteList) { + log.info("不初始化白名单到文件"); + return; + } writeWhiteListFile(); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java index 5315ece1cc..ed07bb067b 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationErrorServiceImpl.java @@ -18,6 +18,7 @@ import com.pamirs.takin.common.util.DateUtils; import com.pamirs.takin.entity.domain.dto.NodeUploadDataDTO; import com.pamirs.takin.entity.domain.entity.ExceptionInfo; +import io.shulie.takin.utils.string.StringUtil; import io.shulie.takin.web.biz.pojo.input.application.ApplicationErrorQueryInput; import io.shulie.takin.web.biz.pojo.output.application.ApplicationErrorOutput; import io.shulie.takin.web.biz.pojo.output.application.ApplicationExceptionOutput; @@ -65,15 +66,15 @@ public List list(ApplicationErrorQueryInput queryRequest // 应用节点相关错误信息 ApplicationErrorOutput nodeErrorResponse = - this.getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); + this.getNodeErrorResponse(tApplicationMnt.getApplicationName(), tApplicationMnt.getNodeNum()); if (nodeErrorResponse != null) { responseList.add(nodeErrorResponse); } //redisKey改造 String appUniqueKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - queryRequest.getApplicationId() + ApplicationServiceImpl.PRADARNODE_KEYSET); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + queryRequest.getApplicationId() + ApplicationServiceImpl.PRADARNODE_KEYSET); Set keys = redisTemplate.opsForSet().members(appUniqueKey); if (keys == null || keys.size() == 0) { return responseList; @@ -92,9 +93,9 @@ public List list(ApplicationErrorQueryInput queryRequest return this.processErrorList(responseList); } - private ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInput queryRequest) { + public ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInput queryRequest) { Response applicationMntResponse = applicationService.getApplicationInfoForError( - String.valueOf(queryRequest.getApplicationId())); + String.valueOf(queryRequest.getApplicationId())); ApplicationDetailResult tApplicationMnt = applicationMntResponse.getData(); if (Objects.isNull(tApplicationMnt)) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, "应用不存在"); @@ -103,11 +104,11 @@ private ApplicationDetailResult ensureApplicationExist(ApplicationErrorQueryInpu } private void putNodeExceptionIfNeeded(List responseList, - ApplicationDetailResult tApplicationMnt) { + ApplicationDetailResult tApplicationMnt) { Integer totalNodeCount = tApplicationMnt.getNodeNum(); Integer onlineNodeCount = 0; List applicationResultList = applicationDAO.getApplicationByName( - Collections.singletonList(tApplicationMnt.getApplicationName())); + Collections.singletonList(tApplicationMnt.getApplicationName())); if (CollectionUtils.isEmpty(applicationResultList)) { log.error("AMDB中应用信息查询结果为空"); } else { @@ -119,16 +120,16 @@ private void putNodeExceptionIfNeeded(List responseList, } if (!totalNodeCount.equals(onlineNodeCount)) { responseList.add(new ApplicationErrorOutput() - .setExceptionId("-") - .setAgentIdList(Collections.singletonList("-")) - .setDescription("在线节点数 与 配置的节点总数 不一致") - .setTime(DateUtils.getNowDateStr()) - .setDetail("设置节点数：" + totalNodeCount + "，在线节点数：" + onlineNodeCount)); + .setExceptionId("-") + .setAgentIdList(Collections.singletonList("-")) + .setDescription("在线节点数 与 配置的节点总数 不一致") + .setTime(DateUtils.getNowDateStr()) + .setDetail("设置节点数：" + totalNodeCount + "，在线节点数：" + onlineNodeCount)); } } private void convertNodeUploadDataList(List responseList, - List nodeUploadDataDTOList) { + List nodeUploadDataDTOList) { nodeUploadDataDTOList.parallelStream().forEach(n -> { NodeUploadDataDTO nodeUploadDataDTO = JSONObject.parseObject(n, NodeUploadDataDTO.class); Map exceptionMap = nodeUploadDataDTO.getSwitchErrorMap(); @@ -143,13 +144,16 @@ private void convertNodeUploadDataList(List responseList log.error("异常转换失败：错误信息: {},异常内容{}", message, e.getMessage()); } ApplicationErrorOutput applicationErrorResponse - = new ApplicationErrorOutput() - .setExceptionId(exceptionInfo != null ? exceptionInfo.getErrorCode() : "web-异常原文显示") - .setAgentIdList(Collections.singletonList(nodeUploadDataDTO.getAgentId())) - .setDescription(exceptionInfo != null ? exceptionInfo.getMessage() : message) - .setDetail(exceptionInfo != null ? exceptionInfo.getDetail() : message) - .setTime(nodeUploadDataDTO.getExceptionTime()); - responseList.add(applicationErrorResponse); + = new ApplicationErrorOutput() + .setExceptionId(exceptionInfo != null ? exceptionInfo.getErrorCode() : "web-异常原文显示") + .setAgentIdList(Collections.singletonList(nodeUploadDataDTO.getAgentId())) + .setDescription(exceptionInfo != null ? exceptionInfo.getMessage() : message) + .setDetail(exceptionInfo != null ? exceptionInfo.getDetail() : message) + .setTime(nodeUploadDataDTO.getExceptionTime()); + if (!StringUtil.equals("探针接入异常", applicationErrorResponse.getDetail()) + || !StringUtil.equals("探针接入异常", applicationErrorResponse.getDescription())) { + responseList.add(applicationErrorResponse); + } } } } @@ -173,8 +177,8 @@ public List getAppException(List appNames) { } //redisKey改造 String appUniqueKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceTenantCode(), - app.getAppId() + ApplicationServiceImpl.PRADAR_SEPERATE_FLAG); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceTenantCode(), + app.getAppId() + ApplicationServiceImpl.PRADAR_SEPERATE_FLAG); Set keys = redisTemplate.keys(appUniqueKey + "*"); if (keys != null) { for (String nodeKey : keys) { @@ -191,7 +195,7 @@ public List getAppException(List appNames) { if (message.contains("errorCode")) { try { ExceptionInfo exceptionInfo = JSONObject.parseObject(message, - ExceptionInfo.class); + ExceptionInfo.class); ApplicationExceptionOutput output = new ApplicationExceptionOutput(); output.setApplicationName(app.getAppName()); output.setAgentIds(Arrays.asList(nodeUploadDataDTO.getAgentId())); @@ -223,13 +227,13 @@ public List getAppException(List appNames) { * @param totalNodeCount 节点数量 * @return 节点错误 */ - private ApplicationErrorOutput getNodeErrorResponse(String applicationName, Integer totalNodeCount) { + public ApplicationErrorOutput getNodeErrorResponse(String applicationName, Integer totalNodeCount) { List applicationResultList = applicationDAO.getApplicationByName( - Collections.singletonList(applicationName)); + Collections.singletonList(applicationName)); ApplicationErrorOutput applicationErrorResponse = null; if (CollectionUtils.isEmpty(applicationResultList) - || !totalNodeCount.equals(applicationResultList.get(0).getInstanceInfo().getInstanceOnlineAmount())) { + || !totalNodeCount.equals(applicationResultList.get(0).getInstanceInfo().getInstanceOnlineAmount())) { applicationErrorResponse = new ApplicationErrorOutput(); applicationErrorResponse.setExceptionId("-"); applicationErrorResponse.setAgentIdList(Collections.singletonList("-")); @@ -255,14 +259,14 @@ private ApplicationErrorOutput getNodeErrorResponse(String applicationName, Inte private List processErrorList(List responseList) { // 按照时间倒序输出 List sortedList = responseList.parallelStream() - .filter(t -> t != null && CharSequenceUtil.isNotBlank(t.getTime())) - .sorted((a1, a2) -> a2.getTime().compareTo(a1.getTime())) - .collect(Collectors.toList()); + .filter(t -> t != null && CharSequenceUtil.isNotBlank(t.getTime())) + .sorted((a1, a2) -> a2.getTime().compareTo(a1.getTime())) + .collect(Collectors.toList()); List noTimeList = responseList.parallelStream() - // 无时间的 - .filter(response -> response != null && CharSequenceUtil.isBlank(response.getTime())) - .collect(Collectors.toList()); + // 无时间的 + .filter(response -> response != null && CharSequenceUtil.isBlank(response.getTime())) + .collect(Collectors.toList()); if (sortedList.isEmpty()) { return noTimeList; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java index 32f2281434..af459df30d 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/application/impl/ApplicationMiddlewareServiceImpl.java @@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -70,9 +71,13 @@ public class ApplicationMiddlewareServiceImpl implements ApplicationMiddlewareSe @Autowired private MiddlewareJarService middlewareJarService; + //默认不处理中间件信息 + @Value("${takin.enable.middlewareFlag:false}") + private boolean middlewareFlag; + @Override public PagingList page( - ListApplicationMiddlewareRequest listApplicationMiddlewareRequest) { + ListApplicationMiddlewareRequest listApplicationMiddlewareRequest) { PageApplicationMiddlewareParam pageApplicationMiddlewareParam = new PageApplicationMiddlewareParam(); BeanUtils.copyProperties(listApplicationMiddlewareRequest, pageApplicationMiddlewareParam); @@ -90,7 +95,7 @@ public PagingList page( // 状态转换 ApplicationMiddlewareStatusEnum applicationMiddlewareStatusEnum = - ApplicationMiddlewareStatusEnum.getByCode(result.getStatus()); + ApplicationMiddlewareStatusEnum.getByCode(result.getStatus()); if (applicationMiddlewareStatusEnum != null) { response.setStatusDesc(applicationMiddlewareStatusEnum.getDesc()); } @@ -104,7 +109,7 @@ public PagingList page( public ApplicationMiddlewareCountResponse countSome(Long applicationId) { PageUtils.clearPageHelper(); List statusMapCountResultList = applicationMiddlewareDAO - .listCountByApplicationIdAndStatusAndGroupByStatus(applicationId, null); + .listCountByApplicationIdAndStatusAndGroupByStatus(applicationId, null); if (statusMapCountResultList.isEmpty()) { return new ApplicationMiddlewareCountResponse(); @@ -112,8 +117,8 @@ public ApplicationMiddlewareCountResponse countSome(Long applicationId) { // 状态统计转为 状态 -> 统计个数 map Map statusAboutCount = statusMapCountResultList.stream() - .collect(Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, - ApplicationMiddlewareStatusAboutCountResult::getCount, (v1, v2) -> v2)); + .collect(Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, + ApplicationMiddlewareStatusAboutCountResult::getCount, (v1, v2) -> v2)); ApplicationMiddlewareCountResponse response = new ApplicationMiddlewareCountResponse(); response.setTotalCount(applicationMiddlewareDAO.countByApplicationIdAndStatus(applicationId, null)); @@ -122,7 +127,7 @@ public ApplicationMiddlewareCountResponse countSome(Long applicationId) { response.setNotSupportedCount(statusAboutCount.get(ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode())); response.setNoneCount(statusAboutCount.get(ApplicationMiddlewareStatusEnum.NONE.getCode())); response.setNoSupportRequiredCount( - statusAboutCount.get(ApplicationMiddlewareStatusEnum.NO_SUPPORT_REQUIRED.getCode())); + statusAboutCount.get(ApplicationMiddlewareStatusEnum.NO_SUPPORT_REQUIRED.getCode())); return response; } @@ -156,7 +161,7 @@ public void compare(Long applicationId) { public List doCompare(List results) { // 转 dto List compareApplicationMiddlewareList = DataTransformUtil.list2list(results, - CompareApplicationMiddlewareDTO.class); + CompareApplicationMiddlewareDTO.class); // 比对 middlewareJarService.appCompare(compareApplicationMiddlewareList); @@ -168,6 +173,9 @@ public List doCompare(List middlewareList = pushMiddlewareRequest.getMiddlewareList(); if (middlewareList.isEmpty()) { return; @@ -195,9 +203,9 @@ public void pushMiddlewareList(PushMiddlewareRequest pushMiddlewareRequest) { // 新的中间件插入 log.info("应用中间件上报 --> 插入上报中间件"); List createApplicationMiddlewareParamList = - this.listCreateApplicationMiddlewareParam(middlewareList, application); + this.listCreateApplicationMiddlewareParam(middlewareList, application); this.isPushError(!applicationMiddlewareDAO.insertBatch(createApplicationMiddlewareParamList), - "应用中间件报错失败!"); + "应用中间件报错失败!"); } catch (Exception e) { // 发生错误, 解锁 @@ -221,20 +229,20 @@ public void pushMiddlewareList(PushMiddlewareRequest pushMiddlewareRequest) { @Override public Map> getApplicationNameAboutStatusCountMap( - List applicationIds) { + List applicationIds) { List statusList = Arrays.asList(ApplicationMiddlewareStatusEnum.NONE.getCode(), - ApplicationMiddlewareStatusEnum.UNKNOWN.getCode(), - ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode()); + ApplicationMiddlewareStatusEnum.UNKNOWN.getCode(), + ApplicationMiddlewareStatusEnum.NOT_SUPPORTED.getCode()); List results = applicationMiddlewareDAO - .listStatusCountByAndGroupByApplicationNameListAndStatus(applicationIds, statusList); + .listStatusCountByAndGroupByApplicationNameListAndStatus(applicationIds, statusList); if (results.isEmpty()) { return Collections.emptyMap(); } return results.stream() - .collect(Collectors.groupingBy(ApplicationMiddlewareStatusAboutCountResult::getApplicationName, - Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, - ApplicationMiddlewareStatusAboutCountResult::getCount))); + .collect(Collectors.groupingBy(ApplicationMiddlewareStatusAboutCountResult::getApplicationName, + Collectors.toMap(ApplicationMiddlewareStatusAboutCountResult::getStatus, + ApplicationMiddlewareStatusAboutCountResult::getCount))); } /** @@ -267,7 +275,7 @@ private void isPushError(boolean condition, String message) { * @return 待新增的中间件对象列表 */ private List listCreateApplicationMiddlewareParam( - List middlewareList, ApplicationDetailResult application) { + List middlewareList, ApplicationDetailResult application) { return middlewareList.stream().map(pushMiddlewareListRequest -> { CreateApplicationMiddlewareParam createParam = new CreateApplicationMiddlewareParam(); createParam.setApplicationId(application.getApplicationId()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java index 1b83c02100..03493f245e 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationPluginsConfigServiceImpl.java @@ -7,6 +7,7 @@ import javax.annotation.Resource; +import cn.hutool.core.util.NumberUtil; import com.baomidou.mybatisplus.core.metadata.IPage; import com.google.common.collect.Lists; import io.shulie.takin.common.beans.page.PagingList; @@ -82,7 +83,7 @@ public PagingList getPageByParam(ApplicationPluginsC if ("-1".equals(configVO.getConfigValue())) { configVO.setConfigValueName("与业务key一致"); } else { - configVO.setConfigValueName(record.getConfigValue() + "小时"); + configVO.setConfigValueName(record.getConfigValue() + "分钟"); } //精度丢失问题 configVO.setApplicationId(record.getApplicationId() + ""); @@ -204,7 +205,7 @@ public Boolean update(ApplicationPluginsConfigParam param) { OperationLogContextHolder.operationType(OpTypes.UPDATE); OperationLogContextHolder.addVars(Vars.APPLICATION_ID,oldEntity.getApplicationId().toString()); OperationLogContextHolder.addVars(Vars.APP_PLUGIN_KEY,oldEntity.getConfigItem()); - OperationLogContextHolder.addVars(Vars.APP_PLUGIN_VALUE,oldEntity.getConfigValue().equals("-1")?"与业务key一致":oldEntity.getConfigValue()+" h"); + OperationLogContextHolder.addVars(Vars.APP_PLUGIN_VALUE,oldEntity.getConfigValue().equals("-1")?"与业务key一致":oldEntity.getConfigValue()+" min"); return true; } @@ -216,9 +217,18 @@ public List getListByParam(ApplicationPluginsConfigP if (Objects.isNull(param.getConfigKey())) { throw new TakinWebException(ExceptionCode.POD_NUM_EMPTY, "configKey为空"); } + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(param.getApplicationName()); + param.setApplicationId(application.getApplicationId()); List list = applicationPluginsConfigDAO.findList(param); if (list != null && !list.isEmpty()) { - return CopyUtils.copyFieldsList(list, ApplicationPluginsConfigVO.class); + List vos = CopyUtils.copyFieldsList(list, ApplicationPluginsConfigVO.class); + vos.forEach(x -> { + if(!"-1".equals(x.getConfigValue())){ + x.setConfigValue(NumberUtil.div(x.getConfigValue(),"60",3).toString()); + } + }); +// vos.forEach(x -> x.setConfigValue(NumberUtil.div(x.getConfigValue(),"60",3).toString())); + return vos; } return null; } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java index 5a497878e5..dc8a30ef85 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ApplicationServiceImpl.java @@ -43,6 +43,7 @@ import io.shulie.takin.web.biz.pojo.input.application.*; import io.shulie.takin.web.biz.pojo.input.whitelist.WhitelistImportFromExcelInput; import io.shulie.takin.web.biz.pojo.openapi.response.application.ApplicationListResponse; +import io.shulie.takin.web.biz.pojo.output.application.ApplicationErrorOutput; import io.shulie.takin.web.biz.pojo.request.activity.ActivityCreateRequest; import io.shulie.takin.web.biz.pojo.request.application.ApplicationListByUpgradeRequest; import io.shulie.takin.web.biz.pojo.request.application.ApplicationNodeOperateProbeRequest; @@ -55,7 +56,9 @@ import io.shulie.takin.web.biz.pojo.response.application.ShadowServerConfigurationResponse; import io.shulie.takin.web.biz.pojo.vo.application.ApplicationDsManageExportVO; import io.shulie.takin.web.biz.service.*; +import io.shulie.takin.web.biz.service.application.ApplicationErrorService; import io.shulie.takin.web.biz.service.application.ApplicationNodeService; +import io.shulie.takin.web.biz.service.application.impl.ApplicationErrorServiceImpl; import io.shulie.takin.web.biz.service.dsManage.DsService; import io.shulie.takin.web.biz.service.linkmanage.LinkGuardService; import io.shulie.takin.web.biz.service.linkmanage.WhiteListService; @@ -119,6 +122,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.annotation.PostConstruct; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.math.BigDecimal; @@ -127,6 +131,7 @@ import java.util.*; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -364,50 +369,57 @@ public Response> getApplicationList(ApplicationQueryRequest @Override public Long getAccessErrorNum() { - List results = applicationDAO.getDashboardAppData(); - // 通过amdb查询状态 - if (results == null || results.size() == 0) { - return 0L; - } - //取应用节点数信息 - List appNameList = results.stream().map(ApplicationDetailResult::getApplicationName).collect( - Collectors.toList()); - List applicationResultList = applicationDAO.getApplicationByName(appNameList); - if (CollectionUtil.isEmpty(applicationResultList)) { - return (long) results.size(); - } - Map> appResultMap = applicationResultList.stream() - .collect(Collectors.groupingBy(ApplicationResult::getAppName)); - //取应用节点版本信息 - ApplicationNodeQueryParam queryParam = new ApplicationNodeQueryParam(); - queryParam.setCurrent(0); - queryParam.setPageSize(99999); - queryParam.setApplicationNames(appNameList); - PagingList applicationNodes = applicationNodeDAO.pageNodes(queryParam); - if (CollectionUtil.isEmpty(applicationResultList)) { - return (long) results.size(); - } - - List applicationNodeResultList = applicationNodes.getList(); - Map> applicationNodeResultMap = applicationNodeResultList - .stream().collect(Collectors.groupingBy(ApplicationNodeResult::getAppName)); - return results.stream().filter(result -> { - List nodeResults = applicationNodeResultMap.get(result.getApplicationName()); - List appResults = appResultMap.get(result.getApplicationName()); - if (CollectionUtils.isEmpty(nodeResults) || CollectionUtils.isEmpty(appResults)) { - return true; - } - if (!appResults.get(0).getInstanceInfo().getInstanceOnlineAmount().equals(result.getNodeNum()) - || nodeResults.stream().map(ApplicationNodeResult::getAgentVersion).distinct().count() > 1) { - return true; - } - // 自身异常 - if (AppAccessStatusEnum.EXCEPTION.getCode().equals(result.getAccessStatus())) { - return true; - } - return false; - }).count(); - } + ApplicationQueryRequestV2 requestV2 = new ApplicationQueryRequestV2(); + requestV2.setAccessStatus(3); + return this.pageApplication(requestV2).getTotal(); + } + +// @Override +// public Long getAccessErrorNum() { +// List results = applicationDAO.getDashboardAppData(); +// // 通过amdb查询状态 +// if (results == null || results.size() == 0) { +// return 0L; +// } +// //取应用节点数信息 +// List appNameList = results.stream().map(ApplicationDetailResult::getApplicationName).collect( +// Collectors.toList()); +// List applicationResultList = applicationDAO.getApplicationByName(appNameList); +// if (CollectionUtil.isEmpty(applicationResultList)) { +// return (long) results.size(); +// } +// Map> appResultMap = applicationResultList.stream() +// .collect(Collectors.groupingBy(ApplicationResult::getAppName)); +// //取应用节点版本信息 +// ApplicationNodeQueryParam queryParam = new ApplicationNodeQueryParam(); +// queryParam.setCurrent(0); +// queryParam.setPageSize(99999); +// queryParam.setApplicationNames(appNameList); +// PagingList applicationNodes = applicationNodeDAO.pageNodes(queryParam); +// if (CollectionUtil.isEmpty(applicationResultList)) { +// return (long) results.size(); +// } +// +// List applicationNodeResultList = applicationNodes.getList(); +// Map> applicationNodeResultMap = applicationNodeResultList +// .stream().collect(Collectors.groupingBy(ApplicationNodeResult::getAppName)); +// return results.stream().filter(result -> { +// List nodeResults = applicationNodeResultMap.get(result.getApplicationName()); +// List appResults = appResultMap.get(result.getApplicationName()); +// if (CollectionUtils.isEmpty(nodeResults) || CollectionUtils.isEmpty(appResults)) { +// return true; +// } +// if (!appResults.get(0).getInstanceInfo().getInstanceOnlineAmount().equals(result.getNodeNum()) +// || nodeResults.stream().map(ApplicationNodeResult::getAgentVersion).distinct().count() > 1) { +// return true; +// } +// // 自身异常 +// if (AppAccessStatusEnum.EXCEPTION.getCode().equals(result.getAccessStatus())) { +// return true; +// } +// return false; +// }).count(); +// } @Override public List getApplicationListVo(ApplicationQueryRequest queryParam) { @@ -487,7 +499,6 @@ public Response getApplicationInfo(String id) { if (tApplicationMnt == null) { return Response.success(new ApplicationVo()); } - // 取应用节点数信息 List applicationResultList = applicationDAO.getApplicationByName( Collections.singletonList(tApplicationMnt.getApplicationName())); @@ -666,7 +677,7 @@ public void uploadAppStatus(NodeUploadDataDTO param) { } @Override - public synchronized void syncApplicationAccessStatus() { + public void syncApplicationAccessStatus() { try { // 应用分页大小 int pageSize = 20; @@ -675,19 +686,16 @@ public synchronized void syncApplicationAccessStatus() { PageBaseDTO pageBaseDTO = new PageBaseDTO(); pageBaseDTO.setPageSize(pageSize); - do { // 分页查询数据库应用列表 List applicationList = applicationDAO.pageFromSync(pageBaseDTO); if (applicationList.isEmpty()) { return; } - // 下一页 pageBaseDTO.setCurrent(pageBaseDTO.getCurrent() + 1); // 赋值查询出的应用数量 applicationNumber = applicationList.size(); - // 收集应用名称 List appNames = applicationList.stream() .map(ApplicationListResult::getApplicationName) @@ -705,6 +713,7 @@ public synchronized void syncApplicationAccessStatus() { // 正常的应用 Set normalApplicationIdSet = new HashSet<>(20); + Map errorInfo = Maps.newHashMap(); // 遍历比对 for (ApplicationListResult application : applicationList) { String applicationName = application.getApplicationName(); @@ -721,6 +730,8 @@ public synchronized void syncApplicationAccessStatus() { || !Objects.equals(amdbApplication.getInstanceInfo().getInstanceOnlineAmount(), nodeNum)) { // amdbApplicationMap 不存在, map.get 不存在, 或者节点数不一致 errorApplicationIdSet.add(applicationId); + errorInfo.put(applicationId, "节点数不一致"); + } else if (!amdbApplicationMap.isEmpty() && (amdbApplication = amdbApplicationMap.get(applicationName)) != null @@ -743,7 +754,7 @@ public synchronized void syncApplicationAccessStatus() { } - this.syncApplicationAccessStatus(applicationList,errorApplicationIdSet); + this.syncApplicationAccessStatus(applicationList, errorApplicationIdSet, errorInfo); } while (applicationNumber == pageSize); // 先执行一遍, 然后如果分页应用数量等于pageSize, 那么查询下一页 @@ -753,18 +764,28 @@ public synchronized void syncApplicationAccessStatus() { log.debug("定时同步应用状态完成!"); } - private void syncApplicationAccessStatus(List applicationList,Set errorApplicationIdSet) { + private void syncApplicationAccessStatus(List applicationList + , Set errorApplicationIdSet, Map errorInfo) { if (CollectionUtils.isNotEmpty(applicationList)) { - applicationList.forEach(app -> { + for (ApplicationListResult app : applicationList) { Map result = applicationDAO.getStatus(app.getApplicationName()); long n = (long) result.get("n"); if (n != 0 || (errorApplicationIdSet.contains(app.getApplicationId()))) { String e = (String) result.get("e"); + if (StringUtils.isBlank(e)) { - String a = (String)result.get("a"); + String a = (String) result.get("a"); + if (StringUtils.isEmpty(a)) { + if (!io.shulie.takin.utils.string.StringUtil + .isEmpty(errorInfo.get(app.getApplicationId()))) { + //节点不一致 + applicationDAO.updateStatus(app.getApplicationId(), e); + } + continue; + } e = "探针接入异常"; if (StringUtils.isNotEmpty(a)) { - e += "，agentId为"+a; + e += "，agentId为" + a; } } applicationDAO.updateStatus(app.getApplicationId(), e); @@ -782,8 +803,10 @@ private void syncApplicationAccessStatus(List application param.setSwitchErrorMap(map); uploadAccessStatus(param); } else { - applicationDAO.updateStatus(app.getApplicationId());} - }); + applicationDAO.updateStatus(app.getApplicationId()); + } + + } } } @@ -1186,8 +1209,8 @@ public String getApplicationNameByApplicationId(Long applicationId) { @Override public void uninstallAllAgent(List appIds) { try { - appIds = this.filterAppIds(appIds,AgentConstants.UNINSTALL); - if (CollectionUtils.isEmpty(appIds)){ + appIds = this.filterAppIds(appIds, AgentConstants.UNINSTALL); + if (CollectionUtils.isEmpty(appIds)) { log.info("所有需要卸载的应用都被过滤掉了"); return; } @@ -1295,6 +1318,9 @@ public List getAllTenantApp(List commo return applicationDAO.getAllTenantApp(commonExtList); } + @Resource + ApplicationErrorServiceImpl applicationErrorService; + @Override public PagingList pageApplication(ApplicationQueryRequestV2 request) { QueryApplicationParam queryApplicationParam = BeanUtil.copyProperties(request, QueryApplicationParam.class); @@ -1312,6 +1338,12 @@ public PagingList pageApplication(ApplicationQueryReq List responseList = records.stream().map(result -> { ApplicationListResponseV2 response = BeanUtil.copyProperties(result, ApplicationListResponseV2.class); response.setId(result.getApplicationId().toString()); + + // 跟应用详情再对比下,同步下状态 + Response vo = this.getApplicationInfo(response.getId()); + if (vo.getSuccess() && vo.getData() != null) { + response.setAccessStatus(vo.getData().getAccessStatus()); + } return response; }).collect(Collectors.toList()); return PagingList.of(responseList, applicationListResultPage.getTotal()); @@ -1340,7 +1372,7 @@ public PagingList listApplicationByUpgrade(App @Override public Response operateCheck(List appIds, String operate) { - if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)){ + if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)) { return Response.fail("参数异常"); } @@ -1356,25 +1388,25 @@ public Response operateCheck(List appIds, String operate) { List appNames = applicationList.stream().map(ApplicationDetailResult::getApplicationName).collect( Collectors.toList()); List applicationNodeProbeResults = applicationNodeProbeDAO.listByAppNameAndOperate(ApplicationNodeProbeOperateEnum.UNINSTALL.getCode(), appNames); - long count = applicationNodeProbeResults == null ? 0 : applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().count(); - if (AgentConstants.UNINSTALL.equals(operate)){ - if (count > 0){ + long count = applicationNodeProbeResults == null ? 0 : applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().count(); + if (AgentConstants.UNINSTALL.equals(operate)) { + if (count > 0) { //构建返回数据 List distinct = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().collect(Collectors.toList()); StringBuilder sb = new StringBuilder(); distinct.forEach(s -> { sb.append(s).append("\n"); }); - return Response.success(String.format("已选择%d个应用,%d个应用已处于卸载状态\n应用名称为:",appIds.size(),count) + sb); - }else { - return Response.success(String.format("已选择%d个应用,点击继续卸载",appIds.size())); + return Response.success(String.format("已选择%d个应用,%d个应用已处于卸载状态\n应用名称为:", appIds.size(), count) + sb); + } else { + return Response.success(String.format("已选择%d个应用,点击继续卸载", appIds.size())); } } - if (AgentConstants.RESUME.equals(operate)){ - if (appIds.size() > count){ + if (AgentConstants.RESUME.equals(operate)) { + if (appIds.size() > count) { //构建返回数据 List result = appNames; - if (count != 0){ + if (count != 0) { List distinct = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName).distinct().collect(Collectors.toList()); result = result.stream().filter(o -> !distinct.contains(o)).collect(Collectors.toList()); } @@ -1382,9 +1414,9 @@ public Response operateCheck(List appIds, String operate) { result.forEach(s -> { sb.append(s).append("\n"); }); - return Response.success(String.format("已选择%d个应用,%d个应用处于非卸载状态\n应用名称为:",appIds.size(), appIds.size() - count) + sb); - }else { - return Response.success(String.format("已选择%d个应用,点击继续",appIds.size())); + return Response.success(String.format("已选择%d个应用,%d个应用处于非卸载状态\n应用名称为:", appIds.size(), appIds.size() - count) + sb); + } else { + return Response.success(String.format("已选择%d个应用,点击继续", appIds.size())); } } return Response.fail("上传的状态当前不支持校验"); @@ -1392,7 +1424,7 @@ public Response operateCheck(List appIds, String operate) { @Override public List filterAppIds(List appIds, String operate) { - if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)){ + if (CollectionUtils.isEmpty(appIds) || StringUtil.isEmpty(operate)) { return null; } @@ -1412,12 +1444,12 @@ public List filterAppIds(List appIds, String operate) { List uninstallAppNames = applicationNodeProbeResults.stream().map(ApplicationNodeProbeResult::getApplicationName) .collect(Collectors.toList()); //需要卸载的数据，需要不存在卸载的数据 - if (AgentConstants.UNINSTALL.equals(operate)){ + if (AgentConstants.UNINSTALL.equals(operate)) { return applicationList.stream().filter(o -> !uninstallAppNames.contains(o.getApplicationName())).map(o -> o.getApplicationId().toString()).collect(Collectors.toList()); } //需要恢复的数据，需要是已经卸载的数据 - if (AgentConstants.RESUME.equals(operate)){ + if (AgentConstants.RESUME.equals(operate)) { return applicationList.stream().filter(o -> uninstallAppNames.contains(o.getApplicationName())).map(o -> o.getApplicationId().toString()).collect(Collectors.toList()); } @@ -2448,8 +2480,8 @@ public Response uploadMiddlewareStatus(Map reques @Override public void resumeAllAgent(List appIds) { try { - appIds = this.filterAppIds(appIds,AgentConstants.RESUME); - if (CollectionUtils.isEmpty(appIds)){ + appIds = this.filterAppIds(appIds, AgentConstants.RESUME); + if (CollectionUtils.isEmpty(appIds)) { log.info("所有需要恢复的应用都被过滤掉了"); return; } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java index 6381118ef4..ee6e18eb8d 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/impl/ShadowConsumerServiceImpl.java @@ -1,12 +1,7 @@ package io.shulie.takin.web.biz.service.impl; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Resource; @@ -137,7 +132,7 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput ApplicationDetailResult application = applicationDAO.getApplicationById(request.getApplicationId()); if (application == null) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, - String.format("应用id:%s对应的应用不存在", request.getApplicationId())); + String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); if (StringUtils.isNotBlank(request.getTopicGroup())) { @@ -148,7 +143,7 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput } if (request.getEnabled() != null) { lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getStatus, - request.getEnabled() ? ShadowConsumerConstants.ENABLE : ShadowConsumerConstants.DISABLE); + request.getEnabled() ? ShadowConsumerConstants.ENABLE : ShadowConsumerConstants.DISABLE); } if (CollectionUtils.isNotEmpty(WebPluginUtils.getQueryAllowUserIdList())) { lambdaQueryWrapper.in(ShadowMqConsumerEntity::getUserId, WebPluginUtils.getQueryAllowUserIdList()); @@ -167,72 +162,72 @@ public PagingList pageMqConsumers(ShadowConsumerQueryInput } private List filterResult(ShadowConsumerQueryInput request, - List totalResult) { + List totalResult) { if (request.getEnabled() != null) { if (request.getEnabled()) { totalResult = totalResult.stream().filter(ShadowConsumerOutput::getEnabled).collect( - Collectors.toList()); + Collectors.toList()); } else { totalResult = totalResult.stream().filter(e -> !e.getEnabled()).collect(Collectors.toList()); } } if (StringUtils.isNotBlank(request.getTopicGroup())) { totalResult = totalResult.stream().filter(e -> e.getTopicGroup().contains(request.getTopicGroup())).collect( - Collectors.toList()); + Collectors.toList()); } if (request.getType() != null) { totalResult = totalResult.stream().filter(e -> e.getType().equals(request.getType())).collect( - Collectors.toList()); + Collectors.toList()); } return totalResult; } private List mergeResult(List amdbResult, - List dbResult) { + List dbResult) { Map amdbMap = new HashMap<>(); Map entityMap = mqConfigTemplateDAO.selectToMapWithNameKey(); if (CollectionUtils.isNotEmpty(amdbResult)) { amdbMap = amdbResult.stream() - .filter(item -> entityMap.containsKey(item.getType())) - .map(e -> { - ShadowConsumerOutput response = new ShadowConsumerOutput(); - response.setUnionId( - MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); - response.setType(e.getType()); - response.setTopicGroup(e.getTopicGroup()); - response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); - response.setGmtCreate(e.getCreateTime()); - response.setGmtUpdate(e.getUpdateTime()); - response.setCanRemove(false); - response.setCanEnableDisable(false); - response.setIsManual(false); - response.setShadowconsumerEnable(String.valueOf(e.getStatus())); - return response; - }) - .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); + .filter(item -> entityMap.containsKey(item.getType())) + .map(e -> { + ShadowConsumerOutput response = new ShadowConsumerOutput(); + response.setUnionId( + MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); + response.setType(e.getType()); + response.setTopicGroup(e.getTopicGroup()); + response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); + response.setGmtCreate(e.getCreateTime()); + response.setGmtUpdate(e.getUpdateTime()); + response.setCanRemove(false); + response.setCanEnableDisable(false); + response.setIsManual(false); + response.setShadowconsumerEnable(String.valueOf(e.getStatus())); + return response; + }) + .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); } Map dbMap = new HashMap<>(dbResult.size()); if (CollectionUtils.isNotEmpty(dbResult)) { dbMap = dbResult.stream() - .filter(item -> entityMap.containsKey(item.getType())) - .map(e -> { - ShadowConsumerOutput response = new ShadowConsumerOutput(); - response.setId(e.getId()); - response.setUnionId( - MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); - response.setType(e.getType()); - response.setTopicGroup(e.getTopicGroup()); - response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); - response.setGmtCreate(e.getCreateTime()); - response.setGmtUpdate(e.getUpdateTime()); - response.setUserId(e.getUserId()); - response.setIsManual(e.getManualTag() == 1); - response.setCanRemove(response.getIsManual()); - response.setShadowconsumerEnable(String.valueOf(e.getStatus())); - WebPluginUtils.fillQueryResponse(response); - return response; - }) - .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); + .filter(item -> entityMap.containsKey(item.getType())) + .map(e -> { + ShadowConsumerOutput response = new ShadowConsumerOutput(); + response.setId(e.getId()); + response.setUnionId( + MD5Util.getMD5(e.getApplicationName() + "#" + e.getTopicGroup() + "#" + e.getType())); + response.setType(e.getType()); + response.setTopicGroup(e.getTopicGroup()); + response.setEnabled(e.getStatus() == ShadowConsumerConstants.ENABLE); + response.setGmtCreate(e.getCreateTime()); + response.setGmtUpdate(e.getUpdateTime()); + response.setUserId(e.getUserId()); + response.setIsManual(e.getManualTag() == 1); + response.setCanRemove(response.getIsManual()); + response.setShadowconsumerEnable(String.valueOf(e.getStatus())); + WebPluginUtils.fillQueryResponse(response); + return response; + }) + .collect(Collectors.toMap(ShadowConsumerOutput::getUnionId, e -> e, (oV, nV) -> nV)); } // 原：在amdb自动梳理的基础上，补充数据库里面的记录，有的话用数据的记录 // 现：在db的基础上，补充amdb自动梳理的数据。 @@ -248,39 +243,51 @@ private List mergeResult(List amdb } private List queryAmdbDefaultEntrances(ShadowConsumerQueryInput request, - String applicationName) { + String applicationName) { List mqTopicGroups = applicationEntranceClient.getMqTopicGroups(applicationName); if (CollectionUtils.isEmpty(mqTopicGroups)) { return Lists.newArrayList(); } if (Objects.nonNull(request.getType())) { mqTopicGroups = mqTopicGroups.stream() - .filter(dto -> dto.getMiddlewareName().equals(request.getType())) - .collect(Collectors.toList()); + .filter(dto -> dto.getMiddlewareName().equals(request.getType())) + .collect(Collectors.toList()); } return mqTopicGroups.stream() - .map(mqTopicGroup -> { - ShadowMqConsumerOutput shadowMqConsumerOutput = new ShadowMqConsumerOutput(); - shadowMqConsumerOutput.setTopicGroup( - mqTopicGroup.getServiceName() + "#" + mqTopicGroup.getMethodName()); - shadowMqConsumerOutput.setType( - MiddlewareTypeGroupEnum.getMiddlewareGroupType(mqTopicGroup.getMiddlewareName()).getType()); - shadowMqConsumerOutput.setApplicationId(request.getApplicationId()); - shadowMqConsumerOutput.setApplicationName(applicationName); - shadowMqConsumerOutput.setStatus(ShadowConsumerConstants.DISABLE); - shadowMqConsumerOutput.setDeleted(ShadowConsumerConstants.LIVED); - // 补充数据 - WebPluginUtils.fillUserData(shadowMqConsumerOutput); - return shadowMqConsumerOutput; - }).collect(Collectors.toList()); + .map(mqTopicGroup -> { + ShadowMqConsumerOutput shadowMqConsumerOutput = new ShadowMqConsumerOutput(); + shadowMqConsumerOutput.setTopicGroup( + mqTopicGroup.getServiceName() + "#" + mqTopicGroup.getMethodName()); + shadowMqConsumerOutput.setType( + MiddlewareTypeGroupEnum.getMiddlewareGroupType(mqTopicGroup.getMiddlewareName()).getType()); + shadowMqConsumerOutput.setApplicationId(request.getApplicationId()); + shadowMqConsumerOutput.setApplicationName(applicationName); + shadowMqConsumerOutput.setStatus(ShadowConsumerConstants.DISABLE); + shadowMqConsumerOutput.setDeleted(ShadowConsumerConstants.LIVED); + // 补充数据 + WebPluginUtils.fillUserData(shadowMqConsumerOutput); + return shadowMqConsumerOutput; + }).collect(Collectors.toList()); } private PagingList splitPage( - ShadowConsumerQueryInput request, - List responses) { + ShadowConsumerQueryInput request, + List responses) { responses.sort((o1, o2) -> { + boolean o1InValid = o1.getGmtCreate() == null; + boolean o2Invalid = o2.getGmtCreate() == null; + boolean bothInvalid = o1InValid && o2Invalid; + if (bothInvalid) { + return 0; + } + if (o1InValid) { + return -1; + } + if (o2Invalid) { + return 1; + } if (o1.getGmtCreate() != null && o2.getGmtCreate() != null) { int firstSort = -o1.getGmtCreate().compareTo(o2.getGmtCreate()); if (firstSort == 0) { @@ -312,10 +319,10 @@ public void createMqConsumers(ShadowConsumerCreateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.CREATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -349,12 +356,12 @@ public void updateMqConsumers(ShadowConsumerUpdateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); // 同名的自己不算 exists = exists.stream().filter(item -> !item.getId().equals(request.getId())).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.UPDATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -371,7 +378,7 @@ public void updateMqConsumers(ShadowConsumerUpdateInput request) { @Override public void importUpdateMqConsumers(ShadowConsumerUpdateInput request) { if (!request.getTopicGroup().contains("#")) { - return; + return; } String[] split = request.getTopicGroup().split("#"); if (split.length != 2) { @@ -382,7 +389,7 @@ public void importUpdateMqConsumers(ShadowConsumerUpdateInput request) { return; } ShadowMqConsumerEntity updateEntity = new ShadowMqConsumerEntity(); - BeanUtils.copyProperties(request,updateEntity); + BeanUtils.copyProperties(request, updateEntity); updateEntity.setTopicGroup(request.getTopicGroup()); updateEntity.setType(request.getType()); updateEntity.setStatus(request.getStatus()); @@ -452,12 +459,12 @@ public void operateMqConsumers(ShadowConsumersOperateInput requests) { } else { OperationLogContextHolder.operationType(OpTypes.DISABLE); List ids = requests.getRequests().stream().map(ShadowConsumerOperateInput::getId).collect( - Collectors.toList()); + Collectors.toList()); LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); lambdaQueryWrapper.in(ShadowMqConsumerEntity::getId, ids); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getDeleted, ShadowConsumerConstants.LIVED); List shadowMqConsumerEntities = shadowMqConsumerMapper.selectList( - lambdaQueryWrapper); + lambdaQueryWrapper); if (CollectionUtils.isNotEmpty(shadowMqConsumerEntities)) { for (ShadowMqConsumerEntity shadowMqConsumerEntity : shadowMqConsumerEntities) { ShadowMqConsumerEntity updateEntity = new ShadowMqConsumerEntity(); @@ -473,23 +480,25 @@ public void operateMqConsumers(ShadowConsumersOperateInput requests) { @Override public List agentSelect(String appName) { + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getDeleted, ShadowConsumerConstants.LIVED); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getStatus, ShadowConsumerConstants.ENABLE); lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getApplicationName, appName); + lambdaQueryWrapper.eq(ShadowMqConsumerEntity::getApplicationId, application.getApplicationId()); List entities = shadowMqConsumerMapper.selectList(lambdaQueryWrapper); if (CollectionUtils.isEmpty(entities)) { return Lists.newArrayList(); } Map> collect = entities.stream() - .filter(t -> { - if (StringUtils.isNotBlank(t.getTopicGroup())) { - String[] topicGroup = t.getTopicGroup().trim().split("#"); - return topicGroup.length == 2; - } else { - return false; - } - }).collect(Collectors.groupingBy(ShadowMqConsumerEntity::getType)); + .filter(t -> { + if (StringUtils.isNotBlank(t.getTopicGroup())) { + String[] topicGroup = t.getTopicGroup().trim().split("#"); + return topicGroup.length == 2; + } else { + return false; + } + }).collect(Collectors.groupingBy(ShadowMqConsumerEntity::getType)); if (MapUtils.isEmpty(collect)) { return Lists.newArrayList(); } @@ -526,7 +535,7 @@ public int allocationUser(ShadowConsumerUpdateUserInput request) { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); queryWrapper.eq(ShadowMqConsumerEntity::getApplicationId, request.getApplicationId()); List shadowMqConsumerEntityList = shadowMqConsumerMapper.selectList( - queryWrapper); + queryWrapper); if (CollectionUtils.isNotEmpty(shadowMqConsumerEntityList)) { for (ShadowMqConsumerEntity entity : shadowMqConsumerEntityList) { entity.setUserId(request.getUserId()); @@ -571,7 +580,7 @@ public List queryMqSupportProgramme(String engName) { @Override @Transactional(rollbackFor = Throwable.class) public void updateMqConsumersV2(ShadowConsumerUpdateInput request) { - request.setTopicGroup(StringUtil.isEmpty(request.getTopicGroup()) ?"":request.getTopicGroup().trim()); + request.setTopicGroup(StringUtil.isEmpty(request.getTopicGroup()) ? "" : request.getTopicGroup().trim()); if (Objects.isNull(request.getId())) { this.createMqConsumersV2(request, false); } else { @@ -587,12 +596,12 @@ public void updateMqConsumersV2(ShadowConsumerUpdateInput request) { throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); // 同名的自己不算 exists = exists.stream().filter(item -> !item.getId().equals(request.getId())).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.UPDATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -623,10 +632,10 @@ public void createMqConsumersV2(ShadowConsumerCreateInput request, Boolean manua throw new RuntimeException(String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } List exists = getExists(request.getTopicGroup(), request.getApplicationId(), - request.getType()); + request.getType()); if (CollectionUtils.isNotEmpty(exists)) { throw new RuntimeException( - String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); + String.format("类型为[%s]，对应的[%s]已存在", request.getType(), request.getTopicGroup())); } OperationLogContextHolder.operationType(OpTypes.CREATE); OperationLogContextHolder.addVars(Vars.CONSUMER_TYPE, request.getType()); @@ -651,7 +660,7 @@ public PagingList pageMqConsumersV2(ShadowConsumerQueryInp ShadowConsumerQueryInput queryInput = Convert.convert(ShadowConsumerQueryInput.class, request); if (application == null) { throw new TakinWebException(TakinWebExceptionEnum.APPLICATION_MANAGE_VALIDATE_ERROR, - String.format("应用id:%s对应的应用不存在", request.getApplicationId())); + String.format("应用id:%s对应的应用不存在", request.getApplicationId())); } LambdaQueryWrapper lambdaQueryWrapper = new LambdaQueryWrapper<>(); if (StringUtils.isNotBlank(request.getTopicGroup())) { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java index 5a8e6b9561..1bb3851ef8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/AppRemoteCallServiceImpl.java @@ -95,6 +95,7 @@ import org.apache.commons.lang3.StringUtils; import org.mockito.internal.util.collections.Sets; import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; /** @@ -131,6 +132,9 @@ public class AppRemoteCallServiceImpl implements AppRemoteCallService { @Resource private ThreadPoolExecutor queryAsyncThreadPool; + @Value("${takin.job.app.limit:50}") + private int appSize = 0; + @PostConstruct public void init() { criticaValue = ConfigServerHelper.getWrapperIntegerValueByKey( @@ -545,18 +549,17 @@ public void syncAmdb() { } List voList = dictionaryDataDAO.getDictByCode("REMOTE_CALL_TYPE"); - int size = 50; // size个轮询一次 - if (results.size() > size) { + if (results.size() > appSize) { int i = 1; boolean loop = true; do { List subList; //批量处理 - if (results.size() > i * size) { - subList = results.subList((i - 1) * size, i * size); + if (results.size() > i * appSize) { + subList = results.subList((i - 1) * appSize, i * appSize); } else { - subList = results.subList((i - 1) * size, results.size()); + subList = results.subList((i - 1) * appSize, results.size()); loop = false; } i++; diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java index 3fe2649083..09141bf539 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/ApplicationApiServiceImpl.java @@ -179,8 +179,10 @@ public Response pullApi(String appName) { @Override public Map> pullApiV1(String appName) { + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); ApplicationApiParam apiParam = new ApplicationApiParam(); apiParam.setAppName(appName); + apiParam.setAppId(application.getApplicationId()); List all = applicationApiDAO.querySimpleWithTenant(apiParam); if (CollectionUtils.isEmpty(all)) { return null; @@ -275,6 +277,12 @@ public Response create(ApiCreateVo vo) { DictionaryCache.getObjectByParam(HTTP_METHOD_TYPE, Integer.parseInt(vo.getMethod())).getLabel()); createParam.setApi(vo.getApi()); createParam.setApplicationName(vo.getApplicationName()); + ApplicationDetailResult applicationDetailResult = applicationDAO.getApplicationByTenantIdAndName(vo.getApplicationName()); + if (applicationDetailResult == null) { + throw new TakinWebException(TakinWebExceptionEnum.AGENT_REGISTER_API, + String.format("应用不存在, 应用名称: %s", vo.getApplicationName())); + } + createParam.setApplicationId(applicationDetailResult.getApplicationId()); createParam.setIsDeleted((byte)0); createParam.setUpdateTime(new Date()); createParam.setCreateTime(new Date()); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java index 3ba0e5712a..b615ec0e55 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/LinkGuardServiceImpl.java @@ -184,7 +184,8 @@ public Response> selectByExample(LinkGuardQueryParam param) { @Override public List agentSelect(String appName) { - List results = linkGuardDAO.selectByAppNameUnderCurrentUser(appName); + ApplicationDetailResult application = applicationDAO.getApplicationByTenantIdAndName(appName); + List results = linkGuardDAO.selectByAppNameUnderCurrentUser(application.getApplicationId()); return results.stream().map(item -> { LinkGuardVo target = new LinkGuardVo(); BeanUtils.copyProperties(item, target); diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java index 2a76dda0ac..4f716d66c8 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/linkmanage/impl/WhiteListFileService.java @@ -79,13 +79,21 @@ public class WhiteListFileService { @Autowired private WhiteListService whiteListService; + // 是否默认初始化白名单 + @Value("${takin.enable.initWhiteList:false}") + private boolean initWhiteList; + @PostConstruct public void init() { + if (!initWhiteList) { + log.info("不初始化白名单到文件"); + return; + } ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( - 0, 1, - 0, TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(1), - r -> new Thread(r, "初始化白名单"), new CallerRunsPolicy()); + 0, 1, + 0, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1), + r -> new Thread(r, "初始化白名单"), new CallerRunsPolicy()); threadPoolExecutor.submit(() -> { log.info("开始初始化白名单"); // 老版本 agent 新版本agent 已转到远程调用模块 @@ -173,10 +181,10 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e Map> whitelistMap; boolean isCheckDuplicateName = Boolean.parseBoolean( - ConfigServerHelper.getValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK)); + ConfigServerHelper.getValueByKey(ConfigServerKeyEnum.TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK)); if (isCheckDuplicateName) { List armdString = agentWhiteLists.stream().map(AgentWhiteList::getInterfaceName).collect( - Collectors.toList()); + Collectors.toList()); existWhite = whiteListService.getExistWhite(armdString, Lists.newArrayList()); // todo 这里再获取一次，感觉很多余，但是不改上面的逻辑，所有这里数据再次从新获取，之后可以重构下 WhitelistSearchParam param = new WhitelistSearchParam(); @@ -184,7 +192,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e param.setUseYn(1); List results = whiteListDAO.getList(param); whitelistMap = results.stream().collect( - Collectors.groupingBy(e -> e.getInterfaceName() + "@@" + e.getType())); + Collectors.groupingBy(e -> e.getInterfaceName() + "@@" + e.getType())); } else { // 获取所有白名单，是否有全局属性 WhitelistSearchParam param = new WhitelistSearchParam(); @@ -193,7 +201,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e param.setUseYn(1); List results = whiteListDAO.getList(param); whitelistMap = results.stream() - .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); + .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); } // 获取所有生效效应，是否有局部应用 @@ -203,7 +211,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e searchParam.setWlistIds(ids); List appResults = whitelistEffectiveAppDao.getList(searchParam); Map> appResultsMap = appResults.stream() - .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); + .collect(Collectors.groupingBy(e -> WhitelistUtil.buildWhiteId(e.getType(), e.getInterfaceName()))); List> wListsResult = Lists.newArrayList(); if (CollectionUtils.isNotEmpty(agentWhiteLists)) { @@ -242,9 +250,9 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e //生效应用 List appLists = appResultsMap.get(id); whiteItemNew.put("appNames", CollectionUtils.isNotEmpty(appLists) ? - appLists.stream().map(WhitelistEffectiveAppResult::getEffectiveAppName).distinct() - .collect(Collectors.toList()) - : Lists.newArrayList()); + appLists.stream().map(WhitelistEffectiveAppResult::getEffectiveAppName).distinct() + .collect(Collectors.toList()) + : Lists.newArrayList()); wListsResult.add(whiteItemNew); } } @@ -261,7 +269,7 @@ public Map queryBlackWhiteList(String appName, TenantCommonExt e private List> getBlackList(TenantCommonExt tenantCommonExt) { List tbLists = tbListMntDao.getAllEnabledBlockList(tenantCommonExt.getTenantId(), - tenantCommonExt.getEnvCode()); + tenantCommonExt.getEnvCode()); if (CollectionUtils.isEmpty(tbLists)) { return Lists.newArrayList(); } @@ -283,10 +291,10 @@ private List getNewBlackList(TenantCommonExt tenantCommonExt) List detailResults = applicationDAO.getApplicationList(param); Map> redisMap = results.stream() - .collect(Collectors.groupingBy(BlacklistResult::getApplicationId)); + .collect(Collectors.groupingBy(BlacklistResult::getApplicationId)); Map> detailResultMap = detailResults.stream() - .collect(Collectors.groupingBy(ApplicationDetailResult::getApplicationId)); + .collect(Collectors.groupingBy(ApplicationDetailResult::getApplicationId)); List vos = Lists.newArrayList(); for (Long id : redisMap.keySet()) { List app = detailResultMap.get(id); @@ -309,20 +317,20 @@ private List getNewBlackList(TenantCommonExt tenantCommonExt) private List agentListWhitelist(TenantCommonExt ext) { List list = applicationDAO.queryIdsByNameAndTenant(Lists.newArrayList(), - ext != null ? ext.getTenantId() : null, ext != null ? ext.getEnvCode() : null); + ext != null ? ext.getTenantId() : null, ext != null ? ext.getEnvCode() : null); if (CollectionUtils.isEmpty(list)) { return Lists.newArrayList(); } List> maps = whiteListDAO.getWhiteListByAppIds(list); return maps.stream().map(it -> { - AgentWhiteList whiteListDTO = new AgentWhiteList(); - whiteListDTO.setInterfaceName((String)it.get("interfaceName")); - whiteListDTO.setType(getType(Integer.parseInt((String)it.get("type")))); - whiteListDTO.setSourceType((String)it.get("type")); - // 过滤生效应用用 - whiteListDTO.setWlistId((Long)it.get("wlistId")); - return whiteListDTO; - } + AgentWhiteList whiteListDTO = new AgentWhiteList(); + whiteListDTO.setInterfaceName((String) it.get("interfaceName")); + whiteListDTO.setType(getType(Integer.parseInt((String) it.get("type")))); + whiteListDTO.setSourceType((String) it.get("type")); + // 过滤生效应用用 + whiteListDTO.setWlistId((Long) it.get("wlistId")); + return whiteListDTO; + } ).collect(Collectors.toList()); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java index c2e1ec0e5c..032b3383d1 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportLocalServiceImpl.java @@ -82,7 +82,15 @@ public static void main(String[] args) { public ReportCountDTO getReportCount(Long reportId) { ReportSummaryResult data = reportSummaryDAO.selectOneByReportId(reportId); if (data == null) { - return new ReportCountDTO(); + ReportCountDTO dto = new ReportCountDTO(); + dto.setRiskMachineCount(0); + dto.setApplicationCount(0); + dto.setMachineCount(0); + dto.setWarnCount(0); + dto.setBottleneckInterfaceCount(0); + dto.setBusinessActivityCount(0); + dto.setNotpassBusinessActivityCount(0); + return dto; } return convert2ReportCountDTO(data); } @@ -427,7 +435,7 @@ private String[] getTpsConfigLength(List configs) { continue; } - if(min == null || min.length > array.getTime().length) { + if (min == null || min.length > array.getTime().length) { min = array.getTime(); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportServiceImpl.java index 4f1f75165b..31784b829c 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportServiceImpl.java @@ -119,7 +119,7 @@ public ReportDetailOutput getReportByReportId(Long reportId) { final ReportDetailByIdReq idReq = new ReportDetailByIdReq() {{ setReportId(reportId); }}; - ReportDetailResp detailResponse = cloudReportApi.detail(idReq); + ReportDetailResp detailResponse = reportApi.getReportByReportId(idReq); // sa超过100 显示100 if (detailResponse.getSa() != null && detailResponse.getSa().compareTo(BigDecimal.valueOf(100)) > 0) { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java index 6f006d01df..18e18c8d66 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/report/impl/ReportTaskServiceImpl.java @@ -1,23 +1,15 @@ package io.shulie.takin.web.biz.service.report.impl; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Date; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - import com.alibaba.fastjson.JSON; - import com.pamirs.takin.entity.domain.dto.report.ReportDetailDTO; import io.shulie.takin.cloud.common.redis.RedisClientUtils; -import io.shulie.takin.cloud.sdk.model.request.report.UpdateReportConclusionReq; -import io.shulie.takin.common.beans.response.ResponseResult; import io.shulie.takin.web.biz.constant.WebRedisKeyConstant; import io.shulie.takin.web.biz.pojo.output.report.ReportDetailOutput; import io.shulie.takin.web.biz.service.DistributedLock; import io.shulie.takin.web.biz.service.report.ReportService; import io.shulie.takin.web.biz.service.report.ReportTaskService; import io.shulie.takin.web.biz.service.risk.ProblemAnalysisService; +import io.shulie.takin.web.biz.threadpool.ThreadPoolUtil; import io.shulie.takin.web.biz.utils.job.JobRedisUtils; import io.shulie.takin.web.common.common.Separator; import io.shulie.takin.web.common.pojo.dto.SceneTaskDto; @@ -28,11 +20,17 @@ import io.shulie.takin.web.ext.entity.tenant.TenantCommonExt; import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.concurrent.TimeUnit; + /** * 1、查询生成中状态的报告（只取一条） * 2、更新报告状态为锁定 @@ -41,7 +39,7 @@ * - 风险机器 * - 容量水位 * 4、更新报告状态为已完成 - * + *

* 配合压测实况：容量水位 * 1、压测中生成报告，执行 机器列表 tps汇总图 机器统计 * 2、压测中时，别忘记解锁 @@ -77,10 +75,6 @@ public class ReportTaskServiceImpl implements ReportTaskService { @Autowired private SceneTaskApi sceneTaskApi; - @Autowired - @Qualifier("collectDataThreadPool") - private ThreadPoolExecutor collectDataThreadPool; - @Autowired @Qualifier("redisTemplate") private RedisTemplate redisTemplate; @@ -100,28 +94,37 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { // 查询报告状态 final ReportDetailOutput report = reportService.getReportById(reportId); if (report == null) { + log.warn("未获取到报告信息,reportId=" + reportId); return false; } // 加锁 // 分布式锁 - String lockKey = JobRedisUtils.getRedisJobReport(WebPluginUtils.traceTenantId(), WebPluginUtils.traceEnvCode(),reportId); + String lockKey = JobRedisUtils.getRedisJobReport(WebPluginUtils.traceTenantId(), WebPluginUtils.traceEnvCode(), reportId); if (!distributedLock.checkLock(lockKey)) { - // 收集数据 单独线程收集 - collectDataThreadPool.execute(collectData(reportId,commonExt,lockKey)); + try { + // 收集数据 单独线程收集 + ThreadPoolUtil.getCollectDataThreadPool().execute(() -> collectData(reportId, commonExt, lockKey)); + } catch (Throwable e) { + // TODO 如果线程池满了，继续走下面的逻辑,否则任务有问题 + log.error("提交线程池任务异常," + ExceptionUtils.getStackTrace(e)); + } } // 压测结束才锁报告 Integer status = report.getTaskStatus(); if (status == null || status != 1) { return false; } + log.info("开始处理报告完成,reportId={},报告状态{}", reportId, status); ReportDetailDTO reportDetailDTO = reportDataCache.getReportDetailDTO(reportId); if (reportDetailDTO == null) { - log.error("未查到报告明细！reportId={}",reportId); + log.error("未查到报告明细！reportId={}", reportId); return false; } Date endTime = reportDetailDTO.getEndTime(); //更新任务的结束时间 - if (!this.updateTaskEndTime(reportId, commonExt, endTime)) { return false; } + if (!this.updateTaskEndTime(reportId, commonExt, endTime)) { + return false; + } // 解除 场景锁 redisClientUtils.delete(SceneTaskUtils.getSceneTaskKey(reportDetailDTO.getSceneId())); @@ -131,8 +134,8 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { redisClientUtils.del(WebRedisKeyConstant.REPORT_WARN_PREFIX + reportId); // 删除key String redisKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); redisClientUtils.del(redisKey); long startTime = System.currentTimeMillis(); Boolean lockResponse = reportService.lockReport(reportId); @@ -147,33 +150,32 @@ public Boolean finishReport(Long reportId, TenantCommonExt commonExt) { log.info("压测结束失败 Report :{}，cloud更新失败", reportId); } - Boolean isLeaked = leakVerifyResultDAO.querySceneIsLeaked(reportId); - if (isLeaked) { - //存在漏数，压测失败，修改压测报告状态 1：通过 0：不通过 - log.info("存在漏数，压测失败，修改压测报告状态:[{}]", reportId); - UpdateReportConclusionReq conclusionReq = new UpdateReportConclusionReq(); - conclusionReq.setId(reportId); - conclusionReq.setConclusion(0); - conclusionReq.setErrorMessage("存在漏数"); - ResponseResult responseResult = sceneTaskApi.updateReportStatus(conclusionReq); - log.info("修改压测报告的结果:[{}]", JSON.toJSONString(responseResult)); - } - +// Boolean isLeaked = leakVerifyResultDAO.querySceneIsLeaked(reportId); +// if (isLeaked) { +// //存在漏数，压测失败，修改压测报告状态 1：通过 0：不通过 +// log.info("存在漏数，压测失败，修改压测报告状态:[{}]", reportId); +// UpdateReportConclusionReq conclusionReq = new UpdateReportConclusionReq(); +// conclusionReq.setId(reportId); +// conclusionReq.setConclusion(0); +// conclusionReq.setErrorMessage("存在漏数"); +// ResponseResult responseResult = sceneTaskApi.updateReportStatus(conclusionReq); +// log.info("修改压测报告的结果:[{}]", JSON.toJSONString(responseResult)); +// } reportDataCache.clearDataCache(reportId); log.info("报告id={}汇总成功，花费时间={}", reportId, (System.currentTimeMillis() - startTime)); - } catch (Exception e) { + } catch (Throwable e) { // log.error("客户端生成报告id={}数据异常:{}", reportId, e.getMessage(), e); //生成报告异常，清空本轮生成表数据 reportClearService.clearReportData(reportId); //压测结束，生成压测报告异常，解锁报告 Boolean unLockReportResult = reportService.unLockReport(reportId); log.error("Unlock Report Success, reportId={} ,unLockReportResult= {}...", reportId, unLockReportResult, - e); + e); } finally { + log.info("报告完成,移除报告缓存,reportId={},报告状态{}", reportId, status); removeReportKey(reportId, commonExt); } - - } catch (Exception e) { + } catch (Throwable e) { log.error("QueryRunningReport Error :{}", e.getMessage()); } return true; @@ -215,34 +217,38 @@ private void removeReportKey(Long reportId, TenantCommonExt commonExt) { * @param reportId 报告 id * @return 可运行 */ - private synchronized Runnable collectData(Long reportId, TenantCommonExt commonExt,String lockKey) { - return () -> { - boolean tryLock = distributedLock.tryLock(lockKey, 1L, 1L, TimeUnit.MINUTES); - if(!tryLock) { + private void collectData(Long reportId, TenantCommonExt commonExt, String lockKey) { + // 有锁,证明任务在处理,不需要等太久,后续会有任务继续处理 + boolean tryLock = distributedLock.tryLock(lockKey, 5L, 60L, TimeUnit.SECONDS); + try { + if (!tryLock) { return; } WebPluginUtils.setTraceTenantContext(commonExt); - - try { - // 检查风险机器 - problemAnalysisService.checkRisk(reportId); - } catch (Exception e) { - log.error("reportId = {}: Check the risk machine,errorMsg= {} ", reportId, e.getMessage()); - } - try { - // 瓶颈处理 - problemAnalysisService.processBottleneck(reportId); - } catch (Exception e) { - log.error("reportId = {}: Bottleneck handling,errorMsg= {} ", reportId, e.getMessage()); - } +// try { +// // 检查风险机器 +// problemAnalysisService.checkRisk(reportId); +// } catch (Exception e) { +// log.error("reportId = {}: Check the risk machine,errorMsg= {} ", reportId, e.getMessage()); +// } +// try { +// // 瓶颈处理 +// problemAnalysisService.processBottleneck(reportId); +// } catch (Exception e) { +// log.error("reportId = {}: Bottleneck handling,errorMsg= {} ", reportId, e.getMessage()); +// } try { //then 报告汇总接口 summaryService.calcReportSummay(reportId); } catch (Exception e) { log.error("reportId = {}: total report ,errorMsg= {}", reportId, e.getMessage()); } + } catch (Throwable e) { + log.error("collectData is fail " + ExceptionUtils.getStackTrace(e)); + } finally { distributedLock.unLockSafely(lockKey); - }; + } + } @Override @@ -253,7 +259,7 @@ public void syncMachineData(Long reportId) { long startTime = System.currentTimeMillis(); problemAnalysisService.syncMachineData(reportId); log.debug("reportId={} syncMachineData success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } @Override @@ -264,7 +270,7 @@ public void calcTpsTarget(Long reportId) { //then tps指标图 summaryService.calcTpsTarget(reportId); log.debug("reportId={} calcTpsTarget success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } @Override @@ -275,6 +281,6 @@ public void calcApplicationSummary(Long reportId) { //汇总应用 机器数 风险机器数 summaryService.calcApplicationSummary(reportId); log.debug("reportId={} calcApplicationSummary success，cost time={}s", reportId, - (System.currentTimeMillis() - startTime) / 1000); + (System.currentTimeMillis() - startTime) / 1000); } } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scene/impl/SceneServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scene/impl/SceneServiceImpl.java index 353c353694..e553b790f2 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scene/impl/SceneServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scene/impl/SceneServiceImpl.java @@ -1,13 +1,7 @@ package io.shulie.takin.web.biz.service.scene.impl; -import java.util.*; -import java.util.stream.Collectors; - -import javax.annotation.Resource; - import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.StrUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.pamirs.takin.entity.domain.dto.linkmanage.ScriptJmxNode; @@ -15,6 +9,7 @@ import io.shulie.takin.cloud.common.utils.JmxUtil; import io.shulie.takin.cloud.entrypoint.scene.mix.SceneMixApi; import io.shulie.takin.cloud.ext.content.enums.NodeTypeEnum; +import io.shulie.takin.cloud.ext.content.enums.SamplerTypeEnum; import io.shulie.takin.cloud.ext.content.script.ScriptNode; import io.shulie.takin.cloud.sdk.model.request.filemanager.FileCreateByStringParamReq; import io.shulie.takin.cloud.sdk.model.request.scenemanage.ScriptAnalyzeRequest; @@ -26,11 +21,7 @@ import io.shulie.takin.web.biz.pojo.request.activity.ActivityCreateRequest; import io.shulie.takin.web.biz.pojo.request.activity.VirtualActivityCreateRequest; import io.shulie.takin.web.biz.pojo.request.filemanage.FileManageUpdateRequest; -import io.shulie.takin.web.biz.pojo.request.linkmanage.BusinessFlowDataFileRequest; -import io.shulie.takin.web.biz.pojo.request.linkmanage.BusinessFlowPageQueryRequest; -import io.shulie.takin.web.biz.pojo.request.linkmanage.BusinessFlowParseRequest; -import io.shulie.takin.web.biz.pojo.request.linkmanage.BusinessFlowUpdateRequest; -import io.shulie.takin.web.biz.pojo.request.linkmanage.SceneLinkRelateRequest; +import io.shulie.takin.web.biz.pojo.request.linkmanage.*; import io.shulie.takin.web.biz.pojo.request.scriptmanage.PluginConfigCreateRequest; import io.shulie.takin.web.biz.pojo.request.scriptmanage.ScriptManageDeployCreateRequest; import io.shulie.takin.web.biz.pojo.request.scriptmanage.ScriptManageDeployUpdateRequest; @@ -59,10 +50,10 @@ import io.shulie.takin.web.data.dao.application.ApplicationDAO; import io.shulie.takin.web.data.dao.filemanage.FileManageDAO; import io.shulie.takin.web.data.dao.linkmanage.SceneDAO; +import io.shulie.takin.web.data.dao.pradar.PradarZkConfigDAO; import io.shulie.takin.web.data.dao.scene.SceneLinkRelateDAO; import io.shulie.takin.web.data.dao.scriptmanage.ScriptManageDAO; import io.shulie.takin.web.data.mapper.mysql.SceneMapper; -import io.shulie.takin.web.data.model.mysql.ApplicationMntEntity; import io.shulie.takin.web.data.model.mysql.SceneEntity; import io.shulie.takin.web.data.param.activity.ActivityExistsQueryParam; import io.shulie.takin.web.data.param.activity.ActivityQueryParam; @@ -75,9 +66,9 @@ import io.shulie.takin.web.data.param.scene.ScenePageQueryParam; import io.shulie.takin.web.data.result.activity.ActivityListResult; import io.shulie.takin.web.data.result.application.ApplicationDetailResult; -import io.shulie.takin.web.data.result.application.ApplicationListResult; import io.shulie.takin.web.data.result.filemanage.FileManageResult; import io.shulie.takin.web.data.result.linkmange.SceneResult; +import io.shulie.takin.web.data.result.pradarzkconfig.PradarZkConfigResult; import io.shulie.takin.web.data.result.scene.SceneLinkRelateResult; import io.shulie.takin.web.data.result.scriptmanage.ScriptManageDeployResult; import io.shulie.takin.web.data.result.scriptmanage.ScriptManageResult; @@ -87,7 +78,6 @@ import io.shulie.takin.web.ext.util.WebPluginUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.SetUtils; import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -95,6 +85,10 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import javax.annotation.Resource; +import java.util.*; +import java.util.stream.Collectors; + /** * @author liyuanba * @date 2021/10/27 10:36 上午 @@ -137,6 +131,10 @@ public class SceneServiceImpl implements SceneService { private ApplicationBusinessActivityService applicationBusinessActivityService; @Resource private ApplicationDAO applicationDAO; + @Resource + private PradarZkConfigDAO pradarZkConfigDAO; + + private static final String SCRIPT_HTTP_REQUEST_MAX_NUM_KEY = "/pradar/config/jmx/http/limit"; @Override public List nodeLinkToBusinessActivity(List nodes, Long sceneId) { @@ -270,6 +268,17 @@ public BusinessFlowDetailResponse parseScriptAndSave(BusinessFlowParseRequest bu } //解析脚本 List data = sceneManageApi.scriptAnalyze(analyzeRequest); + // 限制脚本里http请求个数 + List nodes = getNodes(data); + int httpNodeNum = nodes.stream().filter(scriptNode -> scriptNode.getSamplerType() == SamplerTypeEnum.HTTP).collect(Collectors.toSet()).size(); + PradarZkConfigResult byZkPath = pradarZkConfigDAO.getByZkPath(SCRIPT_HTTP_REQUEST_MAX_NUM_KEY); + if (byZkPath != null) { + int limit = Integer.parseInt(byZkPath.getValue()); + if (limit < httpNodeNum) { + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, String.format("脚本里http请求数量超过开关配置:%s的值%d的限制", SCRIPT_HTTP_REQUEST_MAX_NUM_KEY, limit)); + } + } + List testPlan = JmxUtil.getScriptNodeByType(NodeTypeEnum.TEST_PLAN, data); if (CollectionUtils.isEmpty(testPlan)) { throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, "脚本文件没有解析到测试计划！"); @@ -290,6 +299,7 @@ public BusinessFlowDetailResponse parseScriptAndSave(BusinessFlowParseRequest bu return result; } + @Transactional(rollbackFor = Exception.class) public SceneCreateParam saveBusinessFlow(String testName, List data, FileManageUpdateRequest fileManageCreateRequest, List pluginList) { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scenemanage/impl/SceneTaskServiceImpl.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scenemanage/impl/SceneTaskServiceImpl.java index eae957c4b5..3ca1404526 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scenemanage/impl/SceneTaskServiceImpl.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/scenemanage/impl/SceneTaskServiceImpl.java @@ -87,9 +87,14 @@ import io.shulie.takin.web.common.vo.scene.BaffleAppVO; import io.shulie.takin.web.data.dao.SceneExcludedApplicationDAO; import io.shulie.takin.web.data.dao.application.ApplicationDAO; +import io.shulie.takin.web.data.dao.linkmanage.SceneDAO; +import io.shulie.takin.web.data.dao.pradar.PradarZkConfigDAO; import io.shulie.takin.web.data.result.application.ApplicationDetailResult; import io.shulie.takin.web.data.result.application.ApplicationResult; +import io.shulie.takin.web.data.result.linkmange.SceneResult; +import io.shulie.takin.web.data.result.pradarzkconfig.PradarZkConfigResult; import io.shulie.takin.web.data.util.ConfigServerHelper; +import io.shulie.takin.web.diff.api.report.ReportApi; import io.shulie.takin.web.diff.api.scenemanage.SceneManageApi; import io.shulie.takin.web.diff.api.scenetask.SceneTaskApi; import io.shulie.takin.web.ext.entity.UserExt; @@ -114,6 +119,7 @@ public class SceneTaskServiceImpl implements SceneTaskService { public static final String PRESSURE_REPORT_ID_SCENE_PREFIX = "pressure:reportId:scene:"; + private static final String SCRIPT_HTTP_REQUEST_MAX_NUM_KEY = "/pradar/config/jmx/http/limit"; @Value("${file.upload.script.path:/nfs/takin/script/}") private String scriptFilePath; @@ -158,7 +164,12 @@ public class SceneTaskServiceImpl implements SceneTaskService { private SceneExcludedApplicationDAO sceneExcludedApplicationDAO; @Resource - private CloudReportApi cloudReportApi; + private ReportApi reportApi; + @Autowired + private SceneDAO sceneDAO; + @Autowired + private PradarZkConfigDAO pradarZkConfigDAO; + /** * 是否是预发环境 @@ -166,6 +177,10 @@ public class SceneTaskServiceImpl implements SceneTaskService { @Value("${takin.inner.pre:0}") private int isInnerPre; + // 场景任务延迟时间,多补偿300s + @Value("${takin.task.delayTime:300}") + private Long taskDelayTime; + @Override public void preStop(Long sceneId) { SceneManageIdReq request = new SceneManageIdReq(); @@ -226,47 +241,53 @@ public SceneActionResp startTask(SceneActionParam param) { String errorMsg = Objects.isNull(errorInfo) ? "" : errorInfo.getMsg(); log.error("takin-cloud查询场景信息返回错误，id={},错误信息：{}", param.getSceneId(), errorMsg); throw new TakinWebException(TakinWebExceptionEnum.SCENE_THIRD_PARTY_ERROR, - getCloudMessage(errorInfo.getCode(), errorInfo.getMsg())); + getCloudMessage(errorInfo.getCode(), errorInfo.getMsg())); } String jsonString = JsonHelper.bean2Json(resp.getData()); SceneManageWrapperDTO sceneData = JsonHelper.json2Bean(jsonString, SceneManageWrapperDTO.class); if (null == sceneData) { log.error("takin-cloud查询场景信息返回错误，id={},错误信息：{}", param.getSceneId(), - "sceneData is null! jsonString=" + jsonString); + "sceneData is null! jsonString=" + jsonString); throw new TakinWebException(TakinWebExceptionEnum.SCENE_THIRD_PARTY_ERROR, - "场景，id=" + param.getSceneId() + " 信息为空"); + "场景，id=" + param.getSceneId() + " 信息为空"); } sceneData.setIsAbsoluteScriptPath(FileUtils.isAbsoluteUploadPath(sceneData.getUploadFile(), scriptFilePath)); + // 场景启动前校验脚本中http采样器数量 + String businessFlowId = resp.getData().getBusinessFlowId(); + if(StringUtils.isNotEmpty(businessFlowId)){ //兼容的老脚本数据结构没有这个字段，直接放过 + checkTotalNodeNum(Long.parseLong(businessFlowId)); + } + // 校验该场景是否正在压测中 if (!SceneManageStatusEnum.ifFinished(sceneData.getStatus())) { // if (redisClientUtils.hasKey(SceneTaskUtils.getSceneTaskKey(param.getSceneId()))) { // 正在压测中 throw new TakinWebException(TakinWebExceptionEnum.SCENE_START_STATUS_ERROR, - "场景，id=" + param.getSceneId() + "已启动压测，请刷新页面！"); + "场景，id=" + param.getSceneId() + "已启动压测，请刷新页面！"); } else { // 记录key 过期时长为压测时长 redisClientUtils.setString(SceneTaskUtils.getSceneTaskKey(param.getSceneId()), - DateUtils.getServerTime(), - Integer.parseInt(sceneData.getPressureTestTime().getTime().toString()), TimeUnit.MINUTES); + DateUtils.getServerTime(), + Integer.parseInt(sceneData.getPressureTestTime().getTime().toString()), TimeUnit.MINUTES); } preCheckStart(sceneData); if (sceneData != null && sceneData.getScriptId() != null) { ScriptManageDeployDetailResponse scriptManageDeployDetail = scriptManageService.getScriptManageDeployDetail( - sceneData.getScriptId()); + sceneData.getScriptId()); List pluginConfigDetailResponseList = scriptManageDeployDetail - .getPluginConfigDetailResponseList(); + .getPluginConfigDetailResponseList(); if (CollectionUtils.isNotEmpty(pluginConfigDetailResponseList)) { List pluginIds = pluginConfigDetailResponseList.stream().map(o -> Long.parseLong(o.getName())) - .collect(Collectors.toList()); + .collect(Collectors.toList()); param.setEnginePluginIds(pluginIds); param.setEnginePlugins(pluginConfigDetailResponseList.stream() - .map(detail -> new ScenePluginParam() {{ - setPluginId(Long.parseLong(detail.getName())); - setPluginVersion(detail.getVersion()); - }}).collect(Collectors.toList())); + .map(detail -> new ScenePluginParam() {{ + setPluginId(Long.parseLong(detail.getName())); + setPluginVersion(detail.getVersion()); + }}).collect(Collectors.toList())); } } //填充操作人信息 @@ -302,21 +323,39 @@ public SceneActionResp startTask(SceneActionParam param) { return startResult; } + private void checkTotalNodeNum(Long businessFlowId){ + if(null!=businessFlowId){ + SceneResult result = sceneDAO.getSceneDetail(businessFlowId); + int totalNodeNum = result.getTotalNodeNum(); + PradarZkConfigResult byZkPath = pradarZkConfigDAO.getByZkPath(SCRIPT_HTTP_REQUEST_MAX_NUM_KEY); + if (byZkPath != null) { + int limit = Integer.parseInt(byZkPath.getValue()); + if (limit < totalNodeNum) { + throw new TakinWebException(TakinWebExceptionEnum.SCRIPT_VALIDATE_ERROR, String.format("脚本里http请求数量超过限制，不要超过%d个。", limit)); + } + } + } + } + private void pushTaskToRedis(SceneActionResp startResult, SceneManageWrapperDTO sceneData) { final Long reportId = startResult.getData(); if (reportId != null) { //计算结束时间 if (sceneData.getPressureTestSecond() == null) { throw new TakinWebException(TakinWebExceptionEnum.SCENE_VALIDATE_ERROR, - "获取压测时长失败！压测时长为" + sceneData.getPressureTestSecond()); + "获取压测时长失败！压测时长为" + sceneData.getPressureTestSecond()); } - //兜底时长 2小时 - final LocalDateTime dateTime = LocalDateTime.now().plusHours(2); + /** + * 整体延迟时间,在压测任务结束的时候,增加个5分钟时间,已防止finishjob未跑完， + * 导致任务存在问题(任务停止不了，报告未生成),finishJob会清理此数据 + */ + Long endTime = sceneData.getPressureTestSecond() + taskDelayTime; + final LocalDateTime dateTime = LocalDateTime.now().plusSeconds(endTime); //组装 SceneTaskDto taskDto = new SceneTaskDto(reportId, ContextSourceEnum.JOB_SCENE, dateTime); //任务添加到redis队列 final String reportKeyName = isInnerPre == 1 ? WebRedisKeyConstant.SCENE_REPORTID_KEY_FOR_INNER_PRE - : WebRedisKeyConstant.SCENE_REPORTID_KEY; + : WebRedisKeyConstant.SCENE_REPORTID_KEY; final String reportKey = WebRedisKeyConstant.getReportKey(reportId); redisTemplate.opsForList().leftPush(reportKeyName, reportKey); redisTemplate.opsForValue().set(reportKey, JSON.toJSONString(taskDto)); @@ -377,8 +416,8 @@ public ResponseResult stopTask(SceneActionParam param) { } SceneActionResp resp = response.getData(); String redisKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, resp.getReportId())); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, resp.getReportId())); redisClientUtils.del(redisKey); // 最后删除 return sceneTaskApi.stopTask(req); @@ -410,7 +449,7 @@ public ResponseResult checkStatus(Long sceneId, Long frontRepor final ReportDetailByIdReq idReq = new ReportDetailByIdReq(); idReq.setReportId(reportId); WebPluginUtils.fillCloudUserData(idReq); - final ReportDetailResp report = cloudReportApi.getReportByReportId(idReq); + final ReportDetailResp report = reportApi.getReportByReportId(idReq); if (null != report && !report.getPressureType().equals(0)) { return response; } @@ -421,23 +460,23 @@ public ResponseResult checkStatus(Long sceneId, Long frontRepor return response; } SceneManageWrapperResp wrapperResp = JSONObject.parseObject(JSON.toJSONString(detailResp.getData()), - SceneManageWrapperResp.class); + SceneManageWrapperResp.class); // 缓存压测中的应用 List sceneBusinessActivityRefList = wrapperResp.getBusinessActivityConfig(); //从活动中提取应用ID，去除重复ID List applicationIds = sceneBusinessActivityRefList.stream() - .map(SceneBusinessActivityRefResp::getApplicationIds) - .filter(StringUtils::isNotEmpty) - .flatMap(appIds -> Arrays.stream(appIds.split(",")).map(Long::parseLong)) - .filter(data -> data > 0L).distinct().collect(Collectors.toList()); + .map(SceneBusinessActivityRefResp::getApplicationIds) + .filter(StringUtils::isNotEmpty) + .flatMap(appIds -> Arrays.stream(appIds.split(",")).map(Long::parseLong)) + .filter(data -> data > 0L).distinct().collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(applicationIds)) { List applicationMntList = applicationDAO.getApplicationByIds(applicationIds); List applicationNames = applicationMntList.stream().map(ApplicationDetailResult::getApplicationName) - .collect(Collectors.toList()); + .collect(Collectors.toList()); // 过期时间，根据 压测时间 + 10s String redisKey = CommonUtil.generateRedisKeyWithSeparator(Separator.Separator3, - WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), - String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); + WebPluginUtils.traceTenantAppKey(), WebPluginUtils.traceEnvCode(), + String.format(WebRedisKeyConstant.PTING_APPLICATION_KEY, reportId)); redisClientUtils.set(redisKey, applicationNames, wrapperResp.getPressureTestSecond() + 10); } @@ -496,33 +535,33 @@ public List checkBusinessActivitiesConfig(List businessActivityIds // 校验数据源是否能正常连接 List errorDbMsgList = verifyTaskConfigList.stream() - .map(verifyTaskConfig -> { - DataSourceTestRequest testRequest = new DataSourceTestRequest(); - testRequest.setJdbcUrl(verifyTaskConfig.getJdbcUrl()); - testRequest.setUsername(verifyTaskConfig.getUsername()); - testRequest.setPassword(verifyTaskConfig.getPassword()); - testRequest.setType(verifyTaskConfig.getType()); - return dataSourceService.testConnection(testRequest); - }) - .filter(StringUtils::isNotBlank).collect(Collectors.toList()); + .map(verifyTaskConfig -> { + DataSourceTestRequest testRequest = new DataSourceTestRequest(); + testRequest.setJdbcUrl(verifyTaskConfig.getJdbcUrl()); + testRequest.setUsername(verifyTaskConfig.getUsername()); + testRequest.setPassword(verifyTaskConfig.getPassword()); + testRequest.setType(verifyTaskConfig.getType()); + return dataSourceService.testConnection(testRequest); + }) + .filter(StringUtils::isNotBlank).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(errorDbMsgList)) { return errorDbMsgList.stream().sorted().collect(Collectors.toList()); } // 校验sql是否能正常运行 List errorSqlMsgList = verifyTaskConfigList.stream() - .map(verifyTaskConfig -> { - SqlTestRequest testRequest = new SqlTestRequest(); - testRequest.setDatasourceId(verifyTaskConfig.getDatasourceId()); - testRequest.setSqls(verifyTaskConfig.getSqls()); - return leakSqlService.testSqlConnection(testRequest); - }).filter(StringUtils::isNotBlank).map(msg -> { - if (msg.contains(AppConstants.COMMA)) { - return Arrays.asList(msg.split(AppConstants.COMMA)); - } else { - return Collections.singletonList(msg); - } - }).flatMap(Collection::stream).collect(Collectors.toList()); + .map(verifyTaskConfig -> { + SqlTestRequest testRequest = new SqlTestRequest(); + testRequest.setDatasourceId(verifyTaskConfig.getDatasourceId()); + testRequest.setSqls(verifyTaskConfig.getSqls()); + return leakSqlService.testSqlConnection(testRequest); + }).filter(StringUtils::isNotBlank).map(msg -> { + if (msg.contains(AppConstants.COMMA)) { + return Arrays.asList(msg.split(AppConstants.COMMA)); + } else { + return Collections.singletonList(msg); + } + }).flatMap(Collection::stream).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(errorSqlMsgList)) { return errorSqlMsgList.stream().sorted().collect(Collectors.toList()); @@ -535,7 +574,7 @@ private void preCheckStart(SceneManageWrapperDTO sceneData) { //检查压测开关，压测开关处于关闭状态时禁止压测 String userSwitchStatusForVo = applicationService.getUserSwitchStatusForVo(); if (StringUtils.isBlank(userSwitchStatusForVo) || !userSwitchStatusForVo.equals( - AppSwitchEnum.OPENED.getCode())) { + AppSwitchEnum.OPENED.getCode())) { throw new TakinWebException(TakinWebExceptionEnum.SCENE_START_STATUS_ERROR, "压测开关处于关闭状态，禁止压测"); } @@ -544,7 +583,7 @@ private void preCheckStart(SceneManageWrapperDTO sceneData) { if (CollectionUtils.isEmpty(sceneBusinessActivityRefList)) { log.error("[{}]场景没有配置业务活动", sceneData.getId()); throw new TakinWebException(TakinWebExceptionEnum.SCENE_START_VALIDATE_ERROR, - "启动压测失败，没有配置业务活动，场景ID为" + sceneData.getId()); + "启动压测失败，没有配置业务活动，场景ID为" + sceneData.getId()); } // 业务活动相关检查 @@ -558,7 +597,7 @@ private void preCheckStart(SceneManageWrapperDTO sceneData) { * @param sceneBusinessActivityRefList 业务活动 */ private void checkBusinessActivity(SceneManageWrapperDTO sceneData, - List sceneBusinessActivityRefList) { + List sceneBusinessActivityRefList) { //需求要求，业务验证异常需要详细输出 StringBuilder errorMsg = new StringBuilder(); @@ -567,7 +606,7 @@ private void checkBusinessActivity(SceneManageWrapperDTO sceneData, // 应用相关检查 boolean checkApplication = ConfigServerHelper.getBooleanValueByKey( - ConfigServerKeyEnum.TAKIN_START_TASK_CHECK_APPLICATION); + ConfigServerKeyEnum.TAKIN_START_TASK_CHECK_APPLICATION); if (!CollectionUtils.isEmpty(applicationIds) && checkApplication) { List applicationMntList = applicationDAO.getApplicationByIds(applicationIds); // todo 临时方案，过滤挡板应用 @@ -576,10 +615,10 @@ private void checkBusinessActivity(SceneManageWrapperDTO sceneData, try { List baffleAppVos = JsonHelper.json2List(config.getConfigValue(), BaffleAppVO.class); List appNames = Optional - .of(baffleAppVos.stream() - .filter(appVO -> sceneData.getId() != null && sceneData.getId().equals(appVO.getSceneId())) - .collect(Collectors.toList())) - .map(t -> t.get(0)).map(BaffleAppVO::getAppName).orElse(Lists.newArrayList()); + .of(baffleAppVos.stream() + .filter(appVO -> sceneData.getId() != null && sceneData.getId().equals(appVO.getSceneId())) + .collect(Collectors.toList())) + .map(t -> t.get(0)).map(BaffleAppVO::getAppName).orElse(Lists.newArrayList()); List appIds = Lists.newArrayList(); List tempApps = applicationMntList.stream().filter(app -> { @@ -591,18 +630,18 @@ private void checkBusinessActivity(SceneManageWrapperDTO sceneData, return true; }).collect(Collectors.toList()); List tempAppIds = applicationIds.stream().filter(id -> !appIds.contains(id)).collect( - Collectors.toList()); + Collectors.toList()); applicationMntList = tempApps; applicationIds = tempAppIds; } catch (Exception e) { log.error("场景挡板配置转化异常：配置项：{},配置项内容:{}", ConfigConstants.SCENE_BAFFLE_APP_CONFIG, - config.getConfigValue()); + config.getConfigValue()); } } if (CollectionUtils.isEmpty(applicationMntList) || applicationMntList.size() != applicationIds.size()) { log.error("启动压测失败, 没有找到关联的应用信息，场景ID：{}", sceneData.getId()); throw new TakinWebException(TakinWebExceptionEnum.SCENE_START_VALIDATE_ERROR, - "启动压测失败, 没有找到关联的应用信息，场景ID：" + sceneData.getId()); + "启动压测失败, 没有找到关联的应用信息，场景ID：" + sceneData.getId()); } // 检查应用相关 @@ -644,11 +683,11 @@ public String checkScriptCorrelation(SceneManageWrapperDTO sceneData) { public String checkApplicationCorrelation(List applicationMntList) { // 查询下 应用节点信息 节点不一致 也需要返回异常 List appNames = applicationMntList.stream().map(ApplicationDetailResult::getApplicationName).collect( - Collectors.toList()); + Collectors.toList()); // 从大数据里查出数据 todo 目前大数据不区分客户，所以有可能存在不准确问题 List applicationResultList = applicationDAO.getApplicationByName(appNames); Map> listMap = applicationResultList.stream().collect( - Collectors.groupingBy(ApplicationResult::getAppName)); + Collectors.groupingBy(ApplicationResult::getAppName)); List applicationNameList = applicationMntList.stream().map(application -> { boolean statusError = false; @@ -660,14 +699,14 @@ public String checkApplicationCorrelation(List applicat if (!AppAccessTypeEnum.NORMAL.getValue().equals(application.getAccessStatus())) { log.error("应用[{}]接入状态不是开启状态，当前状态[{}]", application.getApplicationName(), - application.getAccessStatus()); + application.getAccessStatus()); statusError = true; } //判断节点数是否异常 Integer totalNodeCount = application.getNodeNum(); List results = listMap.get(application.getApplicationName()); if (CollectionUtils.isEmpty(results) || !totalNodeCount.equals(results.get(0).getInstanceInfo() - .getInstanceOnlineAmount())) { + .getInstanceOnlineAmount())) { log.error("应用[{}]在线节点数与节点总数不一致", application.getApplicationName()); statusError = true; } @@ -680,7 +719,7 @@ public String checkApplicationCorrelation(List applicat } return String.format("%d个应用状态不正常：%s", applicationNameList.size(), - StringUtils.join(applicationNameList.toArray(), AppConstants.COMMA)) + Constants.SPLIT; + StringUtils.join(applicationNameList.toArray(), AppConstants.COMMA)) + Constants.SPLIT; } @Override @@ -690,7 +729,7 @@ public Long getReportIdFromCache(Long sceneId) { if (reportId == null) { return null; } - return Long.valueOf((Integer)reportId); + return Long.valueOf((Integer) reportId); } @Override @@ -708,13 +747,13 @@ public ResponseResult checkSceneJobStatus(Long sceneId) { private Map> getSceneSla(SceneManageWrapperResp detailResp) { Map> dataMap = Maps.newHashMap(); List stopList = Optional.ofNullable(detailResp.getStopCondition()).orElse( - Lists.newArrayList()); + Lists.newArrayList()); stopList = stopList.stream().filter(data -> data.getRule().getIndexInfo() >= 4).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(stopList)) { dataMap.put("stop", stopList); } List warnList = Optional.ofNullable(detailResp.getWarningCondition()).orElse( - Lists.newArrayList()); + Lists.newArrayList()); warnList = warnList.stream().filter(data -> data.getRule().getIndexInfo() >= 4).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(warnList)) { dataMap.put("warn", warnList); @@ -730,12 +769,12 @@ private Map> getSceneSla(SceneManageWrapperResp de * @return 应用ids */ private List listApplicationIdsFromScene(Long sceneId, - List sceneBusinessActivityRefList) { + List sceneBusinessActivityRefList) { // 从活动中提取应用ID，去除重复ID List applicationIds = sceneBusinessActivityRefList.stream() - .map(SceneBusinessActivityRefDTO::getApplicationIds).filter(StringUtils::isNotEmpty) - .flatMap(appIds -> Arrays.stream(appIds.split(",")) - .map(Long::valueOf)).filter(data -> data > 0L).distinct().collect(Collectors.toList()); + .map(SceneBusinessActivityRefDTO::getApplicationIds).filter(StringUtils::isNotEmpty) + .flatMap(appIds -> Arrays.stream(appIds.split(",")) + .map(Long::valueOf)).filter(data -> data > 0L).distinct().collect(Collectors.toList()); if (applicationIds.isEmpty()) { return Collections.emptyList(); } diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java index ebd1c9123e..4e3b4f68f1 100644 --- a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/service/simplify/ShadowJobConfigService.java @@ -260,7 +260,7 @@ public List queryByAppName(String appName) { query.setOrderBy("id desc"); } query.setApplicationId(tApplicationMnt.getApplicationId()); - query.setUserIds(WebPluginUtils.getQueryAllowUserIdList()); +// query.setUserIds(WebPluginUtils.getQueryAllowUserIdList()); //agent调用不需要 List tShadowJobConfigs = tShadowJobConfigMapper.selectList(query); try { for (TShadowJobConfig tShadowJobConfig : tShadowJobConfigs) { diff --git a/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java new file mode 100644 index 0000000000..fae6be626f --- /dev/null +++ b/takin-web-biz-service/src/main/java/io/shulie/takin/web/biz/threadpool/ThreadPoolUtil.java @@ -0,0 +1,111 @@ +package io.shulie.takin.web.biz.threadpool; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 隔离任务线程池,相互之间不影响，彼此之间不影响 + * + * @author + * @description: TODO + * @date 2022/7/7 9:32 AM + */ +@Component +public class ThreadPoolUtil implements ApplicationContextAware { + private static ApplicationContext applicationContext; + + List syncMachineList = + Arrays.asList("reportMachineThreadPool", + "reportMachineThreadPool_one", + "reportMachineThreadPool_two"); + + List reportTpsList = + Arrays.asList("reportTpsThreadPool", + "reportTpsThreadPool_one", + "reportTpsThreadPool_two"); + + List reportSummaryList = + Arrays.asList("reportSummaryThreadPool", + "reportSummaryThreadPool_one", + "reportSummaryThreadPool_two"); + + List reportFinishList = + Arrays.asList("reportFinishThreadPool", + "reportFinishThreadPool_one", + "reportFinishThreadPool_two"); + + List collectDataList = + Arrays.asList("collectDataThreadPool", + "collectDataThreadPool_one", + "collectDataThreadPool_two", + "collectDataThreadPool_three"); + + private static AtomicLong atomicMachine = new AtomicLong(0); + private static AtomicLong atomicTps = new AtomicLong(0); + private static AtomicLong atomicSummary = new AtomicLong(0); + private static AtomicLong atomicFinish = new AtomicLong(0); + private static AtomicLong atomicCollectData = new AtomicLong(0); + + static List syncMachineDataJobThreadPools = new ArrayList<>(); + static List reportTpsThreadPools = new ArrayList<>(); + static List reportSummaryThreadPools = new ArrayList<>(); + static List reportFinishThreadPools = new ArrayList<>(); + static List collectDataThreadPools = new ArrayList<>(); + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @PostConstruct + public void init() { + syncMachineList.stream().forEach(pool -> { + syncMachineDataJobThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportTpsList.stream().forEach(pool -> { + reportTpsThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportSummaryList.stream().forEach(pool -> { + reportSummaryThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + reportFinishList.stream().forEach(pool -> { + reportFinishThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + + collectDataList.stream().forEach(pool -> { + collectDataThreadPools.add(applicationContext.getBean(pool, ThreadPoolExecutor.class)); + }); + } + + public static ThreadPoolExecutor getSyncMachinePool() { + return syncMachineDataJobThreadPools.get((int) Math.abs(atomicMachine.getAndIncrement() % syncMachineDataJobThreadPools.size())); + } + + public static ThreadPoolExecutor getReportTpsThreadPool() { + return reportTpsThreadPools.get((int) Math.abs(atomicTps.getAndIncrement() % reportTpsThreadPools.size())); + } + + public static ThreadPoolExecutor getReportSummaryThreadPool() { + return reportSummaryThreadPools.get((int) Math.abs(atomicSummary.getAndIncrement() % reportSummaryThreadPools.size())); + } + + public static ThreadPoolExecutor getReportFinishThreadPool() { + return reportFinishThreadPools.get((int) Math.abs(atomicFinish.getAndIncrement() % reportFinishThreadPools.size())); + } + + public static ThreadPoolExecutor getCollectDataThreadPool() { + return collectDataThreadPools.get((int) Math.abs(atomicCollectData.getAndIncrement() % collectDataThreadPools.size())); + } +} diff --git a/takin-web-common/pom.xml b/takin-web-common/pom.xml index d2433f1701..fb3b78c6f8 100644 --- a/takin-web-common/pom.xml +++ b/takin-web-common/pom.xml @@ -137,6 +137,18 @@ module-pradar-core provided + + + com.dangdang + elastic-job-lite-lifecycle + + + org.eclipse.jetty.aggregate + jetty-all-server + + + 2.1.5 + diff --git a/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java b/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java index bec23eebba..3e9cd07d46 100644 --- a/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java +++ b/takin-web-common/src/main/java/io/shulie/takin/web/common/enums/config/ConfigServerKeyEnum.java @@ -26,7 +26,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 是否执行添加linux用户的操作 */ TAKIN_FILE_OPS_SCRIPT_DEPLOY_USER_ENABLE("file.ops_script.deploy_user_enable", - "takin.file.ops_script.deploy_user_enable", NO), + "takin.file.ops_script.deploy_user_enable", NO), /** * 白名单长度校验 @@ -37,7 +37,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 是否开启白名单重名校验 */ TAKIN_WHITE_LIST_DUPLICATE_NAME_CHECK("whitelist.duplicate.name.check", "takin.white_list.duplicate.name.check", - YES), + YES), /** * 白名单文件 */ @@ -122,7 +122,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 压测引擎上传时间间隔 单位毫秒 */ TAKIN_PRESSURE_MACHINE_UPLOAD_INTERVAL_TIME("pressure.machine.upload.interval.time", - "takin.pressure.machine.upload.interval.time", NO), + "takin.pressure.machine.upload.interval.time", NO), /** * 脚本调试支持的 rpcType mq 下的 @@ -146,7 +146,7 @@ public enum ConfigServerKeyEnum implements AppConstants { * 等待时间 单位秒 */ TAKIN_PRADAR_SWITCH_PROCESSING_WAIT_TIME("pradar.switch.processing.wait.time", - "takin.pradar.switch.processing.wait.time", NO), + "takin.pradar.switch.processing.wait.time", NO), /** * 租户 id @@ -278,6 +278,10 @@ public enum ConfigServerKeyEnum implements AppConstants { */ TAKIN_TENANT_DEFAULT_PASSWORD("", "takin.tenant.default.password", YES), + /** + * 是否同步数据到zk + */ + TAKIN_ENABLE_SYN_CONFIG("", "takin.enable.syn.config", NO), ; /** diff --git a/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java b/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java new file mode 100644 index 0000000000..b67ec0830c --- /dev/null +++ b/takin-web-common/src/main/java/io/shulie/takin/web/common/util/JVMUtil.java @@ -0,0 +1,81 @@ +package io.shulie.takin.web.common.util; + +import java.io.OutputStream; +import java.lang.management.*; + +import static java.lang.Thread.State.*; + +/** + * @author zhouyuan + * @Date 2021年7月4号 + */ +public class JVMUtil { + public static void jstack(OutputStream stream) throws Exception { + ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); + for (ThreadInfo threadInfo : threadMxBean.dumpAllThreads(true, true)) { + stream.write(getThreadDumpString(threadInfo).getBytes()); + } + } + + private static String getThreadDumpString(ThreadInfo threadInfo) { + StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" + + " Id=" + threadInfo.getThreadId() + " " + + threadInfo.getThreadState()); + if (threadInfo.getLockName() != null) { + sb.append(" on " + threadInfo.getLockName()); + } + if (threadInfo.getLockOwnerName() != null) { + sb.append(" owned by \"" + threadInfo.getLockOwnerName() + + "\" Id=" + threadInfo.getLockOwnerId()); + } + if (threadInfo.isSuspended()) { + sb.append(" (suspended)"); + } + if (threadInfo.isInNative()) { + sb.append(" (in native)"); + } + sb.append('\n'); + int i = 0; + + StackTraceElement[] stackTrace = threadInfo.getStackTrace(); + MonitorInfo[] lockedMonitors = threadInfo.getLockedMonitors(); + for (; i < stackTrace.length && i < 32; i++) { + StackTraceElement ste = stackTrace[i]; + sb.append("\tat ").append(ste.toString()); + sb.append('\n'); + if (i == 0 && threadInfo.getLockInfo() != null) { + Thread.State ts = threadInfo.getThreadState(); + if (BLOCKED.equals(ts)) { + sb.append("\t- blocked on ").append(threadInfo.getLockInfo()); + sb.append('\n'); + } else if (WAITING.equals(ts) || TIMED_WAITING.equals(ts)) { + sb.append("\t- waiting on ").append(threadInfo.getLockInfo()); + sb.append('\n'); + } + } + + for (MonitorInfo mi : lockedMonitors) { + if (mi.getLockedStackDepth() == i) { + sb.append("\t- locked ").append(mi); + sb.append('\n'); + } + } + } + if (i < stackTrace.length) { + sb.append("\t..."); + sb.append('\n'); + } + + LockInfo[] locks = threadInfo.getLockedSynchronizers(); + if (locks.length > 0) { + sb.append("\n\tNumber of locked synchronizers = " + locks.length); + sb.append('\n'); + for (LockInfo li : locks) { + sb.append("\t- " + li); + sb.append('\n'); + } + } + sb.append('\n'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/takin-web-data/pom.xml b/takin-web-data/pom.xml index 39b8aba2a3..a17324d6ab 100644 --- a/takin-web-data/pom.xml +++ b/takin-web-data/pom.xml @@ -106,10 +106,6 @@ org.aspectj aspectjrt - - io.shulie.takin - takin-web-common - diff --git a/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java b/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java index eca42ba673..cfcc63c796 100644 --- a/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java +++ b/takin-web-data/src/main/java/com/pamirs/takin/entity/domain/query/ApplicationApiParam.java @@ -10,4 +10,6 @@ public class ApplicationApiParam extends BaseTenantBean { private String appName; + + private Long appId; } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationApiDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationApiDAOImpl.java index dcc4e623ac..82dd54b930 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationApiDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/ApplicationApiDAOImpl.java @@ -45,6 +45,7 @@ public int insert(ApplicationApiCreateParam param) { entity.setTenantId(param.getTenantId()); entity.setUserId(param.getUserId()); entity.setIsAgentRegiste(param.getIsAgentRegiste()); + entity.setApplicationId(param.getApplicationId()); return apiManageMapper.insert(entity); } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java index 4017d503c9..c102ccd0a4 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/LinkGuardDAO.java @@ -41,4 +41,6 @@ public interface LinkGuardDAO extends IService { */ void updateAppName(Long applicationId, String appName); + List selectByAppNameUnderCurrentUser(Long appId); + } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java index e425f74aa3..5cbe2cdb1c 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/AppRemoteCallDAOImpl.java @@ -16,6 +16,7 @@ package io.shulie.takin.web.data.dao.application.impl; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -209,7 +210,7 @@ private LambdaQueryWrapper getAppRemoteCallEntityLambdaQuer if (param.getIsSynchronize() != null) { lambdaQueryWrapper.eq(AppRemoteCallEntity::getIsSynchronize, param.getIsSynchronize()); } - lambdaQueryWrapper.orderByDesc(AppRemoteCallEntity::getGmtModified); + //lambdaQueryWrapper.orderByDesc(AppRemoteCallEntity::getGmtModified);优化慢sql return lambdaQueryWrapper; } @@ -309,7 +310,24 @@ public List getAllRecord() { LambdaQueryWrapper lambdaQueryWrapper = this.getLambdaQueryWrapper() .eq(AppRemoteCallEntity::getIsDeleted, 0); - List list = list(lambdaQueryWrapper); + //改分页查,优化慢sql + List list = new ArrayList<>(); + List listTemp; + int i=1; + boolean doFlag; + do { + Page page = new Page<>(i, 10000); + IPage entityPageInfo = this.page(page, lambdaQueryWrapper); + if (CollectionUtils.isNotEmpty(entityPageInfo.getRecords())) { + doFlag = true; + listTemp = entityPageInfo.getRecords(); + list.addAll(listTemp); + i++; + }else{ + doFlag = false; + } + }while (doFlag); + if (list.isEmpty()) { return Collections.emptyList(); } diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/InterfaceTypeMainDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/InterfaceTypeMainDAOImpl.java index e22ba57ce6..170411fc14 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/InterfaceTypeMainDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/InterfaceTypeMainDAOImpl.java @@ -1,9 +1,6 @@ package io.shulie.takin.web.data.dao.application.impl; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -21,6 +18,11 @@ @Repository public class InterfaceTypeMainDAOImpl extends ServiceImpl implements InterfaceTypeMainDAO, MPUtil { + + + Map InterfaceTypeMap = new HashMap<>(); + + @Override public InterfaceTypeMainEntity selectByName(String typeName) { @@ -34,7 +36,16 @@ public InterfaceTypeMainEntity selectById(Long id) { @Override public InterfaceTypeMainEntity selectByOrder(Integer order) { - return selectByOrder(order, true); + /** + * fix: 顺丰2023/03/09邮件中提到改sql查询将频繁 + * 先从map中查询，如果map中没有就查询数据库,然后将查到的数据再放入到map中 + */ + InterfaceTypeMainEntity entity = InterfaceTypeMap.get(order); + if(entity == null){ + entity = selectByOrder(order, true); + InterfaceTypeMap.put(order,entity); + } + return entity; } @Override @@ -72,6 +83,7 @@ public void checkUnique(InterfaceTypeMainEntity entity) { @Override public void addEntity(InterfaceTypeMainEntity entity) { this.save(entity); + InterfaceTypeMap.put(entity.getValueOrder(),entity); } @Override diff --git a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java index 93a5b2f06b..193a2ab0e2 100644 --- a/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java +++ b/takin-web-data/src/main/java/io/shulie/takin/web/data/dao/application/impl/LinkGuardDAOImpl.java @@ -93,4 +93,25 @@ public void updateAppName(Long applicationId, String appName) { this.update(wrapper); } + @Override + public List selectByAppNameUnderCurrentUser(Long appId) { + List results = new ArrayList<>(); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + if (WebPluginUtils.checkUserPlugin()) { + wrapper.eq(LinkGuardEntity::getTenantId, WebPluginUtils.traceTenantId()); + } + wrapper.eq(LinkGuardEntity::getApplicationId, appId); + List linkGuardEntities = this.getBaseMapper().selectList(wrapper); + if (CollectionUtils.isEmpty(linkGuardEntities)) { + return results; + } + return linkGuardEntities.stream().map(item -> { + LinkGuardResult target = new LinkGuardResult(); + BeanUtils.copyProperties(item, target); + // 启动 + target.setIsEnable(item.getIsEnable() == GuardEnableConstants.GUARD_ENABLE); + return target; + }).collect(Collectors.toList()); + } + } diff --git a/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml b/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml index d141111bf7..f0dc437ac8 100644 --- a/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml +++ b/takin-web-data/src/main/resources/mappers/ApplicationApiManageMapper.xml @@ -170,6 +170,9 @@ and APPLICATION_NAME = #{appName} + + and APPLICATION_ID = #{appId} +