Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: add monitorparams in ula #806

Merged
merged 1 commit into from
Feb 27, 2024
Merged

refactor: add monitorparams in ula #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ...boot/src/main/java/io/holoinsight/server/home/bootstrap/HoloinsightHomeConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import io.holoinsight.server.home.biz.service.EnvironmentService;
import io.holoinsight.server.home.biz.service.TenantInitService;
import io.holoinsight.server.home.biz.service.UserinfoVerificationService;
import io.holoinsight.server.home.biz.service.impl.AlarmHistoryServiceImpl;
import io.holoinsight.server.home.biz.service.impl.AlertRuleServiceImpl;
import io.holoinsight.server.home.biz.service.impl.DefaultEnvironmentServiceImpl;
import io.holoinsight.server.home.biz.service.impl.DefaultTenantInitServiceImpl;
import io.holoinsight.server.home.biz.service.impl.UserinfoVerificationServiceImpl;
Expand Down Expand Up @@ -110,6 +112,11 @@ public ProductCtlService productCtlService() {
return new ProductCtlServiceImpl();
}

@Bean
public AlertRuleServiceImpl alertRuleService() {
return new AlertRuleServiceImpl();
}

@Bean
public ParameterSecurityService alertSecurityService() {
return new ParameterSecurityServiceImpl();
Expand Down
2 changes: 2 additions & 0 deletions ...common/src/main/java/io/holoinsight/server/home/common/service/RequestContextAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public interface RequestContextAdapter {

<T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant, String workspace);

void tenantAdapt(String tenant, String workspace);

<T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant);

<T> void queryWrapperWorkspaceAdapt(QueryWrapper<T> queryWrapper, String workspace);
Expand Down
5 changes: 5 additions & 0 deletions ...on/src/main/java/io/holoinsight/server/home/common/service/RequestContextAdapterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ public <T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String ten
}
}

@Override
public void tenantAdapt(String tenant, String workspace) {

}

@Override
public <T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant) {
if (queryWrapper != null && StringUtils.isNotBlank(tenant)) {
Expand Down
42 changes: 38 additions & 4 deletions ...e/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@
*/
package io.holoinsight.server.home.dal.model.dto.conf;

import io.holoinsight.server.agg.v1.core.conf.FillZero;
import io.holoinsight.server.agg.v1.core.conf.JoinMeta;
import io.holoinsight.server.agg.v1.core.conf.OutputItem.Topn;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.util.CollectionUtils;

import java.io.Serializable;
Expand Down Expand Up @@ -44,13 +48,18 @@ public class CollectMetric implements Serializable {

public List<String> tags;

public List<String> refTags;


public List<Metric> metrics;

/****************************** 后置过滤 ******************************/
/**
* after filters
*/
public List<AfterFilter> afterFilters;

/****************************** 日志采样 ******************************/
public Boolean logSample;
public List<LogSampleRule> logSampleRules;

Expand All @@ -60,13 +69,11 @@ public class CollectMetric implements Serializable {
// Logs that exceed 4096 are truncated
public Integer sampleMaxLength = 4096;

/****************************** 预计算 ******************************/
// pre calculate, 是否开启预计算
public Boolean calculate;

/**
* agg 预聚合指标表名称，非前端传入，是后端自动生成
*/
public String aggTableName;
public LogCalculate logCalculate;

// 单机明细数据是否存储
public Boolean notStorage;
Expand Down Expand Up @@ -112,6 +119,33 @@ public static class LogSampleRule implements Serializable {

}

@Data
public static class LogCalculate {

/**
* agg 预聚合指标表名称，非前端传入，是后端自动生成
*/
public String aggTableName;

public FillZero fillZero;

/**
* topN 指标计算
*/
public Topn topn;

// join 元数据
public Boolean joinMeta;
public JoinMetaConfig joinMetaConfig;

}

@EqualsAndHashCode(callSuper = true)
@Data
public static class JoinMetaConfig extends JoinMeta {

}

public boolean checkLogPattern() {
if (!CollectionUtils.isEmpty(metrics)) {
for (Metric metric : metrics) {
Expand Down
16 changes: 15 additions & 1 deletion ...er/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/AggTaskUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.holoinsight.server.agg.v1.core.conf.AggFunc;
import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig;
import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig.Mode;
import io.holoinsight.server.agg.v1.core.conf.FillZero;
import io.holoinsight.server.agg.v1.core.conf.From;
import io.holoinsight.server.agg.v1.core.conf.FromConfigs;
import io.holoinsight.server.agg.v1.core.conf.FromMetrics;
Expand Down Expand Up @@ -145,7 +146,7 @@ public static Output buildOutput(CollectMetric collectMetric) {

OutputItem outputItem = new OutputItem();
outputItem.setType(OUTPUT_STORAGE_ENGINE);
outputItem.setName(collectMetric.getAggTableName());
outputItem.setName(collectMetric.getLogCalculate().getAggTableName());

List<OutputField> fields = new ArrayList<>();
OutputField outputField = new OutputField();
Expand All @@ -154,6 +155,11 @@ public static Output buildOutput(CollectMetric collectMetric) {
fields.add(outputField);
outputItem.setFields(fields);

if (null != collectMetric.getLogCalculate().getTopn()
&& collectMetric.getLogCalculate().getTopn().isEnabled()) {
outputItem.setTopn(collectMetric.getLogCalculate().getTopn());
}

List<OutputItem> items = new ArrayList<>();
items.add(outputItem);

Expand All @@ -164,4 +170,12 @@ public static Output buildOutput(CollectMetric collectMetric) {
public static List<PartitionKey> buildPartition(CollectMetric collectMetric) {
return new ArrayList<>();
}

public static FillZero buildFillZero(CollectMetric collectMetric) {
if (null != collectMetric.getLogCalculate().getFillZero()
&& collectMetric.getLogCalculate().getFillZero().isEnabled()) {
return collectMetric.getLogCalculate().getFillZero();
}
return new FillZero();
}
}
130 changes: 76 additions & 54 deletions ...ome/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io.holoinsight.server.home.dal.model.dto.conf.Translate.TranslateTransform;
import io.holoinsight.server.registry.model.Elect;
import io.holoinsight.server.registry.model.Elect.RefIndex;
import io.holoinsight.server.registry.model.Elect.RefMeta;
import io.holoinsight.server.registry.model.Elect.RefName;
import io.holoinsight.server.registry.model.Elect.TransFormFilter;
import io.holoinsight.server.registry.model.Elect.TransFormFilterAppend;
Expand Down Expand Up @@ -665,76 +666,97 @@ public static GroupBy buildGroupBy(LogParse logParse, ExtraConfig extraConfig,
return buildLogPatternParse(logParse, extraConfig);
}
List<String> tags = collectMetric.getTags();
List<String> refTags = collectMetric.getRefTags();

GroupBy groupBy = new GroupBy();
groupBy.setGroups(new ArrayList<>());
if (CollectionUtils.isEmpty(tags)) {
if (CollectionUtils.isEmpty(tags) && CollectionUtils.isEmpty(refTags)) {
return groupBy;
}
tags.forEach(t -> {
if (!CollectionUtils.isEmpty(tags)) {
tags.forEach(t -> {

SplitCol dim = splitColMap.get(dimColType).get(t);
if (null == dim) {
return;
}
Rule rule = dim.rule;
SplitCol dim = splitColMap.get(dimColType).get(t);
if (null == dim) {
return;
}
Rule rule = dim.rule;

Elect elect = new Elect();
Elect elect = new Elect();

switch (logParse.splitType) {
case leftRight:
Elect.LeftRight leftRight = new Elect.LeftRight();
leftRight.setLeft(rule.left);
leftRight.setLeftIndex(rule.leftIndex);
leftRight.setRight(rule.right);
switch (logParse.splitType) {
case leftRight:
Elect.LeftRight leftRight = new Elect.LeftRight();
leftRight.setLeft(rule.left);
leftRight.setLeftIndex(rule.leftIndex);
leftRight.setRight(rule.right);

elect.setType("leftRight");
elect.setLeftRight(leftRight);
break;
elect.setType("leftRight");
elect.setLeftRight(leftRight);
break;

case separator:
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
break;
case regexp:
if (StringUtils.isNotEmpty(rule.regexpName)) {
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.regexpName);
} else if (rule.pos != null) {
case separator:
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
}
break;
case json:
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.jsonPathSyntax);
break;
default:
break;
}
break;
case regexp:
if (StringUtils.isNotEmpty(rule.regexpName)) {
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.regexpName);
} else if (rule.pos != null) {
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
}
break;
case json:
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.jsonPathSyntax);
break;
default:
break;
}

if (null != rule.translate && !CollectionUtils.isEmpty(rule.translate.transforms)) {
Transform transform = new Transform();
transform.setFilters(convertTransFormFilters(rule.translate.transforms));
elect.setTransform(transform);
}
if (null != rule.translate && !CollectionUtils.isEmpty(rule.translate.transforms)) {
Transform transform = new Transform();
transform.setFilters(convertTransFormFilters(rule.translate.transforms));
elect.setTransform(transform);
}

if (StringUtils.isNotEmpty(rule.defaultValue)) {
elect.setDefaultValue(rule.defaultValue);
}
if (StringUtils.isNotEmpty(rule.defaultValue)) {
elect.setDefaultValue(rule.defaultValue);
}

GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
}

if (!CollectionUtils.isEmpty(refTags)) {
refTags.forEach(t -> {

Elect elect = new Elect();
elect.setType("refMeta");
elect.setRefMeta(new RefMeta());
elect.getRefMeta().setName(t);

GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
}

Map<String, SplitCol> extendColTypes = splitColMap.get(extendColType);
if (!CollectionUtils.isEmpty(extendColTypes)) {
Expand Down
12 changes: 8 additions & 4 deletions ...ice/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,9 @@ public void onEvent(CustomPluginDTO customPluginDTO) {
String tableName = String.format("%s_%s", name, customPluginDTO.id);
sqlTaskMaps.put(tableName, sqlTask);

if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.calculate) {
if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.getCalculate()) {
AggTask aggTask = buildAggTask(collectMetric, customPluginDTO);
aggTaskMaps.put(collectMetric.aggTableName, aggTask);
aggTaskMaps.put(collectMetric.logCalculate.aggTableName, aggTask);
}
}

Expand Down Expand Up @@ -221,6 +221,7 @@ private AggTask buildAggTask(CollectMetric collectMetric, CustomPluginDTO custom
AggTaskUtil.buildGroupBy(collectMetric, tenantInitService.getAggDefaultGroupByTags()));
aggTask.setWindow(AggTaskUtil.buildWindow(customPluginDTO.getPeriodType().dataUnitMs));
aggTask.setOutput(AggTaskUtil.buildOutput(collectMetric));
aggTask.setFillZero(AggTaskUtil.buildFillZero(collectMetric));
return aggTask;
}

Expand Down Expand Up @@ -277,7 +278,7 @@ private void saveMetricInfo(CustomPluginDTO customPluginDTO) {

for (CollectMetric collectMetric : collectMetrics) {
saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, false);
if (Boolean.TRUE == collectMetric.calculate) {
if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.getCalculate()) {
saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, true);
}
}
Expand All @@ -289,7 +290,7 @@ private void saveMetricByCollectMetric(CustomPluginDTO customPluginDTO,
String tableName = collectMetric.getTableName();
String targetTable = collectMetric.getTargetTable();
if (isAgg) {
targetTable = collectMetric.getAggTableName();
targetTable = collectMetric.logCalculate.getAggTableName();
}

try {
Expand Down Expand Up @@ -325,6 +326,9 @@ private void saveMetricByCollectMetric(CustomPluginDTO customPluginDTO,
if (!isAgg && !CollectionUtils.isEmpty(collectMetric.tags)) {
tags.addAll(collectMetric.tags);
}
if (!isAgg && !CollectionUtils.isEmpty(collectMetric.refTags)) {
tags.addAll(collectMetric.refTags);
}
metricInfoDTO.setTags(tags);
metricInfoDTO.setRef(String.valueOf(customPluginDTO.getId()));
MetricInfoDTO db = metricInfoService.queryByMetric(metricInfoDTO.getTenant(),
Expand Down
Loading
Loading