Skip to content

Commit

Permalink
fix: fix multilog plugin of integration (#780)
Browse files Browse the repository at this point in the history
  • Loading branch information
jsy1001de authored Jan 22, 2024
1 parent b97d555 commit 6884cf1
Show file tree
Hide file tree
Showing 7 changed files with 288 additions and 200 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import io.holoinsight.server.home.dal.model.dto.GaeaCollectConfigDTO.GaeaCollectRange;
import io.holoinsight.server.home.dal.model.dto.IntegrationGeneratedDTO;
import io.holoinsight.server.home.dal.model.dto.IntegrationPluginDTO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
Expand All @@ -35,6 +36,7 @@
* @version 1.0: IntegrationGeneratedUpdateListener.java, Date: 2023-07-26 Time: 10:57
*/
@Component
@Slf4j
public class IntegrationGeneratedUpdateListener {

@Autowired
Expand All @@ -51,15 +53,25 @@ void register() {
@Subscribe
@AllowConcurrentEvents
public void onEvent(IntegrationGeneratedDTO integrationGeneratedDTO) {
if (CollectionUtils.isEmpty(integrationGeneratedDTO.config))
return;
IntegrationPluginDTO integrationPluginDTO = convertIntegrationPlugin(integrationGeneratedDTO);

if (StringUtils.isBlank(integrationPluginDTO.json)
|| StringUtils.isBlank(integrationPluginDTO.type))
return;
List<Long> upsert = integrationPluginUpdateListener.upsertGaea(integrationPluginDTO);
notify(upsert);
log.info("[integration_generated][{}][{}] convert start", integrationGeneratedDTO.getProduct(),
integrationGeneratedDTO.getId());

try {
if (CollectionUtils.isEmpty(integrationGeneratedDTO.config))
return;
IntegrationPluginDTO integrationPluginDTO = convertIntegrationPlugin(integrationGeneratedDTO);

if (StringUtils.isBlank(integrationPluginDTO.json)
|| StringUtils.isBlank(integrationPluginDTO.type))
return;
List<Long> upsert = integrationPluginUpdateListener.upsertGaea(integrationPluginDTO);
notify(upsert);
} catch (Throwable t) {
log.error("[integration_plugin][{}][{}] convert error, {}",
integrationGeneratedDTO.getProduct(), integrationGeneratedDTO.getId(), t.getMessage(), t);
}
log.info("[integration_generated][{}][{}] convert end", integrationGeneratedDTO.getProduct(),
integrationGeneratedDTO.getId());
}

private IntegrationPluginDTO convertIntegrationPlugin(IntegrationGeneratedDTO generatedDTO) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import io.holoinsight.server.home.dal.model.dto.IntegrationPluginDTO;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -29,6 +30,7 @@
* @version 1.0: IntegrationPluginUpdateListener.java, v 0.1 2022年06月08日 8:40 下午 xiangwanpeng Exp $
*/
@Component
@Slf4j
public class IntegrationPluginUpdateListener {

@Autowired
Expand All @@ -50,24 +52,34 @@ void register() {
@Subscribe
@AllowConcurrentEvents
public void onEvent(IntegrationPluginDTO integrationPluginDTO) {
boolean needUpsertGaea = isClassicPlugin(integrationPluginDTO.getProduct())
|| checkActionType(integrationPluginDTO, null, this.pluginRepository);
if (needUpsertGaea) {
List<Long> upsert = upsertGaea(integrationPluginDTO);
notify(upsert);
} else if (this.pluginRepository.isHostingPlugin(integrationPluginDTO.type)) {
HostingPlugin hostingPlugin = (HostingPlugin) this.pluginRepository
.getTemplate(integrationPluginDTO.type, integrationPluginDTO.version);
if (integrationPluginDTO.status) {
hostingPlugin.apply(integrationPluginDTO);
} else {
hostingPlugin.disable(integrationPluginDTO);
}
} else if ("OpenAiPlugin".equals(integrationPluginDTO.getProduct())) {
if (!integrationPluginDTO.status) {
this.openAiService.unload(integrationPluginDTO.tenant);
log.info("[integration_plugin][{}][{}] convert start", integrationPluginDTO.getProduct(),
integrationPluginDTO.getId());
try {
boolean needUpsertGaea = isClassicPlugin(integrationPluginDTO.getProduct())
|| checkActionType(integrationPluginDTO, null, this.pluginRepository);
if (needUpsertGaea) {
List<Long> upsert = upsertGaea(integrationPluginDTO);
notify(upsert);
} else if (this.pluginRepository.isHostingPlugin(integrationPluginDTO.type)) {
HostingPlugin hostingPlugin = (HostingPlugin) this.pluginRepository
.getTemplate(integrationPluginDTO.type, integrationPluginDTO.version);
if (integrationPluginDTO.status) {
hostingPlugin.apply(integrationPluginDTO);
} else {
hostingPlugin.disable(integrationPluginDTO);
}
} else if ("OpenAiPlugin".equals(integrationPluginDTO.getProduct())) {
if (!integrationPluginDTO.status) {
this.openAiService.unload(integrationPluginDTO.tenant);
}
}
} catch (Throwable t) {
log.error("[integration_plugin][{}][{}] convert error, {}", integrationPluginDTO.getProduct(),
integrationPluginDTO.getId(), t.getMessage(), t);
}

log.info("[integration_plugin][{}][{}] convert end", integrationPluginDTO.getProduct(),
integrationPluginDTO.getId());
}

public List<Long> upsertGaea(IntegrationPluginDTO integrationPluginDTO) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,18 @@
import io.holoinsight.server.home.biz.common.GaeaConvertUtil;
import io.holoinsight.server.home.biz.common.GaeaSqlTaskUtil;
import io.holoinsight.server.home.biz.plugin.config.LogPluginConfig;
import io.holoinsight.server.home.common.util.MonitorException;
import io.holoinsight.server.home.dal.model.dto.CustomPluginPeriodType;
import io.holoinsight.server.home.dal.model.dto.IntegrationPluginDTO;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric.AfterFilter;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric.Metric;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf.ExtraConfig;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf.SplitCol;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf.SpmCols;
import io.holoinsight.server.home.dal.model.dto.conf.Filter;
import io.holoinsight.server.home.dal.model.dto.conf.FilterType;
import io.holoinsight.server.home.dal.model.dto.conf.LogParse;
import io.holoinsight.server.home.dal.model.dto.conf.LogPath;
import io.holoinsight.server.common.J;
Expand All @@ -32,6 +38,7 @@

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -127,6 +134,8 @@ public List<LogPlugin> genPluginList(IntegrationPluginDTO integrationPluginDTO)
CustomPluginConf customPluginConf =
J.fromJson(J.toJson(config.conf), new TypeToken<CustomPluginConf>() {}.getType());

addSpmColInPluginConf(customPluginConf);

if (CollectionUtils.isEmpty(customPluginConf.collectMetrics))
continue;
CustomPluginPeriodType periodType = config.periodType;
Expand Down Expand Up @@ -184,6 +193,10 @@ public String getPrefix() {
return null;
}

public void addSpmColInPluginConf(CustomPluginConf conf) {

}

@Autowired
private MetricInfoService metricInfoService;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
/*
* Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
*/

package io.holoinsight.server.home.biz.plugin.core;

import io.holoinsight.server.home.common.util.MonitorException;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric.AfterFilter;
import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric.Metric;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf.SplitCol;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf.SpmCols;
import io.holoinsight.server.home.dal.model.dto.conf.FilterType;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

/**
* @author jsy1001de
* @version 1.0: LogPluginUtil.java, Date: 2024-01-19 Time: 15:18
*/
public class LogPluginUtil {


public static void addSpmCols(CustomPluginConf conf) {

Boolean spm = checkSpmConditions(conf);

List<CollectMetric> newCollectMetrics = new ArrayList<>();
int cols = 0;
if (!CollectionUtils.isEmpty(conf.collectMetrics)) {
for (CollectMetric collectMetric : conf.collectMetrics) {
if (null == collectMetric.spm || Boolean.FALSE == collectMetric.spm) {
if (spm == Boolean.TRUE
&& Arrays.asList("total", "success", "fail", "cost", "successPercent")
.contains(collectMetric.tableName)) {
continue;
}
newCollectMetrics.add(collectMetric);
continue;
}

if (spm == Boolean.FALSE) {
continue;
}
newCollectMetrics.add(collectMetric);
cols++;
}
}

// add spm metrics
if (spm == Boolean.TRUE && cols == 0) {
SpmCols spmCols = conf.spmCols;

List<String> tags = new ArrayList<>();
List<SplitCol> splitCols = conf.splitCols;
if (!CollectionUtils.isEmpty(splitCols)) {
splitCols.forEach(splitCol -> {
if (splitCol.colType.equalsIgnoreCase("DIM")) {
tags.add(splitCol.name);
}
});
}

newCollectMetrics.add(genTotalCollectMetric(spmCols, tags));
newCollectMetrics.add(genSuccessCollectMetric(spmCols, tags));
newCollectMetrics.add(genFailCollectMetric(spmCols, tags));
newCollectMetrics.add(genCostCollectMetric(spmCols, tags));
newCollectMetrics.add(genSuccessPercentCollectMetric(tags));
}

conf.setCollectMetrics(newCollectMetrics);

}


private static Boolean checkSpmConditions(CustomPluginConf conf) {
if (null == conf || null == conf.spm) {
return Boolean.FALSE;
} else if (Boolean.FALSE == conf.spm) {
return Boolean.FALSE;
}

if (null == conf.spmCols) {
throw new MonitorException("spmCols is null");
}

SpmCols spmCols = conf.spmCols;
if (StringUtils.isBlank(spmCols.resultKey)) {
throw new MonitorException("resultCols is null");
}
if (CollectionUtils.isEmpty(spmCols.successValue)) {
throw new MonitorException("successValue is null");
}
if (StringUtils.isBlank(spmCols.costKey)) {
throw new MonitorException("costCols is null");
}

return Boolean.TRUE;
}

private static CollectMetric genTotalCollectMetric(SpmCols spmCols, List<String> tags) {
CollectMetric collectMetric = new CollectMetric();
collectMetric.setTableName("total");
collectMetric.setMetricType("count");
collectMetric.setTags(tags);
collectMetric.setSpm(true);
Metric metric = new Metric();
metric.setFunc("count");
metric.setName("value");
if (StringUtils.isNotBlank(spmCols.countKey)) {
collectMetric.setMetricType("select");
metric.setFunc("sum");
metric.setName(spmCols.countKey);
}
collectMetric.setMetrics(Collections.singletonList(metric));
return collectMetric;
}

private static CollectMetric genSuccessCollectMetric(SpmCols spmCols, List<String> tags) {
CollectMetric collectMetric = new CollectMetric();
collectMetric.setTableName("success");
collectMetric.setMetricType("count");
collectMetric.setTags(tags);
collectMetric.setSpm(true);
Metric metric = new Metric();
metric.setFunc("count");
metric.setName("value");
if (StringUtils.isNotBlank(spmCols.countKey)) {
collectMetric.setMetricType("select");
metric.setFunc("sum");
metric.setName(spmCols.countKey);
}
collectMetric.setMetrics(Collections.singletonList(metric));

AfterFilter afterFilter = new AfterFilter();
afterFilter.setName(spmCols.resultKey);
afterFilter.setFilterType(FilterType.IN);
afterFilter.setValues(spmCols.successValue);
collectMetric.setAfterFilters(Collections.singletonList(afterFilter));

return collectMetric;
}

private static CollectMetric genFailCollectMetric(SpmCols spmCols, List<String> tags) {
CollectMetric collectMetric = new CollectMetric();
collectMetric.setTableName("fail");
collectMetric.setMetricType("count");
collectMetric.setTags(tags);
collectMetric.setSpm(true);
Metric metric = new Metric();
metric.setFunc("count");
metric.setName("value");
if (StringUtils.isNotBlank(spmCols.countKey)) {
collectMetric.setMetricType("select");
metric.setFunc("sum");
metric.setName(spmCols.countKey);
}
collectMetric.setMetrics(Collections.singletonList(metric));

AfterFilter afterFilter = new AfterFilter();
afterFilter.setName(spmCols.resultKey);
afterFilter.setFilterType(FilterType.NOT_IN);
afterFilter.setValues(spmCols.successValue);
collectMetric.setAfterFilters(Collections.singletonList(afterFilter));

return collectMetric;
}

private static CollectMetric genCostCollectMetric(SpmCols spmCols, List<String> tags) {
CollectMetric collectMetric = new CollectMetric();
collectMetric.setTableName("cost");
collectMetric.setMetricType("select");
collectMetric.setTags(tags);
collectMetric.setSpm(true);
Metric metric = new Metric();
metric.setFunc("avg");
metric.setName(spmCols.costKey);
collectMetric.setMetrics(Collections.singletonList(metric));

return collectMetric;
}

private static CollectMetric genSuccessPercentCollectMetric(List<String> tags) {
CollectMetric collectMetric = new CollectMetric();
collectMetric.setTableName("successPercent");
collectMetric.setMetricType("count");
collectMetric.setTags(tags);
collectMetric.setSpm(true);
Metric metric = new Metric();
metric.setFunc("count");
metric.setName("value");
collectMetric.setMetrics(Collections.singletonList(metric));


return collectMetric;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package io.holoinsight.server.home.biz.plugin.core;

import io.holoinsight.server.home.biz.plugin.model.PluginModel;
import io.holoinsight.server.home.dal.model.dto.conf.CustomPluginConf;
import org.springframework.stereotype.Component;

/**
Expand All @@ -14,4 +15,8 @@
@PluginModel(name = "io.holoinsight.server.plugin.MultiLogPlugin", version = "1")
public class MultiLogPlugin extends LogPlugin {
public MultiLogPlugin() {}

public void addSpmColInPluginConf(CustomPluginConf conf) {
LogPluginUtil.addSpmCols(conf);
}
}
Loading

0 comments on commit 6884cf1

Please sign in to comment.