Skip to content

Commit

Permalink
sql query
Browse files Browse the repository at this point in the history
  • Loading branch information
saimu.msm committed Aug 10, 2023
1 parent d286036 commit 77704b0
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 0 deletions.
3 changes: 3 additions & 0 deletions ...mon/common-service/src/main/java/io/holoinsight/server/common/model/DataQueryRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import lombok.Data;

import java.util.List;
import java.util.Map;

/**
*
Expand Down Expand Up @@ -55,6 +56,8 @@ public static class QueryDataSource {
// instead of post-calculation
public boolean apmMaterialized;
public String ql;

public Map<String /* column name */, String /* expression */> select;
}

@Data
Expand Down
3 changes: 3 additions & 0 deletions .../common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package io.holoinsight.server.common.service;

import io.holoinsight.server.common.dao.entity.MetaDataDictValue;
import io.holoinsight.server.common.dao.entity.MetricInfo;
import io.holoinsight.server.query.grpc.QueryProto.QueryRequest;

import java.util.Map;
Expand All @@ -17,4 +18,6 @@ public class SuperCache {

public Map<String /* type */, Map<String /* k */, MetaDataDictValue>> metaDataDictValueMap;
public Map<String, QueryRequest> expressionMetricList;

public Map<String /* metric table */, MetricInfo> metricInfoMap;
}
40 changes: 40 additions & 0 deletions .../common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,20 @@
*/
package io.holoinsight.server.common.service;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.holoinsight.server.common.config.ProdLog;
import io.holoinsight.server.common.config.ScheduleLoadTask;
import io.holoinsight.server.common.dao.entity.MetricInfo;
import io.holoinsight.server.common.dao.mapper.MetricInfoMapper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.CollectionUtils;

import javax.annotation.Resource;
import java.util.HashMap;
import java.util.Map;

/**
*
Expand All @@ -23,6 +33,9 @@ public class SuperCacheService extends ScheduleLoadTask {
@Autowired
private MetricInfoService metricInfoService;

@Resource
private MetricInfoMapper metricInfoMapper;

public SuperCache getSc() {
return sc;
}
Expand All @@ -33,6 +46,7 @@ public void load() throws Exception {
SuperCache sc = new SuperCache();
sc.metaDataDictValueMap = metaDictValueService.getMetaDictValue();
sc.expressionMetricList = metricInfoService.querySpmList();
sc.metricInfoMap = queryMetricInfoByPage();
this.sc = sc;
ProdLog.info("[SuperCache] load end");
}
Expand All @@ -46,4 +60,30 @@ public int periodInSeconds() {
public String getTaskName() {
return "SuperCacheService";
}


private Map<String, MetricInfo> queryMetricInfoByPage() {
QueryWrapper<MetricInfo> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("deleted", 0);

int current = 1;
Map<String /* metric table */, MetricInfo> map = new HashMap<>();
int size = 1000;
do {
Page<MetricInfo> page = new Page<>(current++, size);
Page<MetricInfo> result = this.metricInfoMapper.selectPage(page, queryWrapper);

if (result == null || CollectionUtils.isEmpty(result.getRecords())) {
break;
}

for (MetricInfo metricInfo : result.getRecords()) {
if (StringUtils.isEmpty(metricInfo.getMetricTable())) {
continue;
}
map.put(metricInfo.getMetricTable(), metricInfo);
}
} while (current < 1000);
return map;
}
}
114 changes: 114 additions & 0 deletions ...ome/home-web/src/main/java/io/holoinsight/server/home/web/controller/QueryFacadeImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import com.google.protobuf.util.JsonFormat;
import io.holoinsight.server.common.LatchWork;
import io.holoinsight.server.common.UtilMisc;
import io.holoinsight.server.common.dao.entity.MetricInfo;
import io.holoinsight.server.common.service.SuperCacheService;
import io.holoinsight.server.common.threadpool.CommonThreadPools;
import io.holoinsight.server.home.biz.common.MetaDictUtil;
import io.holoinsight.server.home.biz.service.TenantInitService;
Expand Down Expand Up @@ -59,10 +61,13 @@
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

Expand All @@ -84,6 +89,9 @@ public class QueryFacadeImpl extends BaseFacade {
@Autowired
private TenantInitService tenantInitService;

@Autowired
private SuperCacheService superCacheService;

@PostMapping
public JsonResult<QueryResponse> query(@RequestBody DataQueryRequest request) {

Expand Down Expand Up @@ -471,6 +479,10 @@ public QueryProto.QueryRequest convertRequest(DataQueryRequest request) {
if ((System.currentTimeMillis() - d.start) < 80000L) {
d.start -= 60000L;
}
MetricInfo metricInfo = getMetricInfo(d.getMetric());
if (isCeresdb4Storage(metricInfo)) {
parseQl(d, metricInfo);
}

QueryProto.Datasource.Builder datasourceBuilder = QueryProto.Datasource.newBuilder();
toProtoBean(datasourceBuilder, d);
Expand All @@ -487,6 +499,108 @@ public QueryProto.QueryRequest convertRequest(DataQueryRequest request) {
return builder.build();
}



protected void parseQl(QueryDataSource d, MetricInfo metricInfo) {
Set<String> tags = new HashSet<>(J.toList(metricInfo.getTags()));
StringBuilder select = new StringBuilder("select ");
List<String> selects = new ArrayList<>();
if (CollectionUtils.isEmpty(d.select)) {
selects.add(" count(1) as value ");
} else {
for (Map.Entry<String /* column name */, String /* expression */> entry : d.select
.entrySet()) {
String columnName = entry.getKey();
String expression = entry.getValue();
if (StringUtils.isEmpty(expression)) {
selects.add("`" + columnName + "`");
} else {
selects.add(expression + " as " + "`" + columnName + "`");
}
}
}
select.append(String.join(" , ", selects));
select.append(" from ").append(d.metric);
select.append(" where `period` <= ") //
.append(d.end) //
.append(" and `period` >= ") //
.append(d.start);
if (!CollectionUtils.isEmpty(d.filters)) {
for (QueryFilter filter : d.filters) {
if (!tags.contains(filter.name)) {
continue;
}
switch (filter.type) {
case "literal_or":
select.append(" and `").append(filter.name).append("` in ('")
.append(String.join("','", Arrays.asList(filter.value.split("\\|")))).append("')");
break;
case "not_literal_or":
select.append(" and ").append(filter.name).append(" not in ('")
.append(String.join("','", Arrays.asList(filter.value.split("\\|")))).append("')");
break;
case "wildcard":
select.append(" and `").append(filter.name).append("` like '").append(filter.value)
.append("'");
break;
case "regexp":
select.append(" and `").append(filter.name).append("` REGEXP '").append(filter.value)
.append("'");
break;
case "not_regexp_match":
select.append(" and `").append(filter.name).append("` NOT REGEXP '")
.append(filter.value).append("'");
break;
case "literal":
select.append(" and `").append(filter.name).append("` = '").append(filter.value)
.append("'");
break;
case "not_literal":
select.append(" and `").append(filter.name).append("` <> '").append(filter.value)
.append("'");
break;
}
}
}
if (!CollectionUtils.isEmpty(d.groupBy)) {
List<String> gbList = new ArrayList<>();
for (String gb : d.groupBy) {
if (!tags.contains(gb)) {
continue;
}
gbList.add(gb);
}
if (!CollectionUtils.isEmpty(gbList)) {
select.append(" group by `")//
.append(String.join("` , `", gbList)) //
.append("`");
}

}
select.append(" order by `period` acs");
log.info("parse sql {}", select);
d.setQl(select.toString());
}

private MetricInfo getMetricInfo(String metric) {
if (CollectionUtils.isEmpty(this.superCacheService.getSc().metricInfoMap)
|| StringUtils.isEmpty(metric)) {
return null;
}
return this.superCacheService.getSc().metricInfoMap.get(metric);
}

private boolean isCeresdb4Storage(MetricInfo metricInfo) {
if (metricInfo == null) {
return false;
}
if (StringUtils.isNotEmpty(metricInfo.getStorageTenant())
&& StringUtils.equals(metricInfo.getStorageTenant(), "ceresdb4")) {
return true;
}
return false;
}

private long getInterval(String downsample) {
long interval = 60000L;
switch (downsample) {
Expand Down
51 changes: 51 additions & 0 deletions ...home-web/src/test/java/io/holoinsight/server/home/web/controller/QueryFacadeImplTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Alipay.com Inc. Copyright (c) 2004-2018 All Rights Reserved.
*/

package io.holoinsight.server.home.web.controller;

import io.holoinsight.server.common.J;
import io.holoinsight.server.common.dao.entity.MetricInfo;
import io.holoinsight.server.common.model.DataQueryRequest;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Test;

import java.util.Arrays;

import static org.junit.Assert.*;

/**
* @author masaimu
* @version 2023-08-10 14:56:00
*/
public class QueryFacadeImplTest {

@Test
public void name() {
String json =
"{\"start\":1691644303000,\"end\":1691647903000,\"name\":\"a\",\"metric\":\"k8s_pod_mem_util\",\"groupBy\":[\"app\",\"hostname\",\"workspace\",\"pod\",\"ip\",\"namespace\"],\"filters\":[{\"type\":\"literal_or\",\"name\":\"app\",\"value\":\"holoinsight-server|aaaaa\"},{\"type\":\"literal\",\"name\":\"应用ID\",\"value\":\"111111111\"},{\"type\":\"literal\",\"name\":\"fake应用ID\",\"value\":\"22222222\"}]}";
DataQueryRequest.QueryDataSource queryDataSource =
J.fromJson(json, DataQueryRequest.QueryDataSource.class);
MetricInfo metricInfo = new MetricInfo();
metricInfo.setTags(J.toJson(Arrays.asList("app", "应用ID")));
QueryFacadeImpl queryFacade = new QueryFacadeImpl();
queryFacade.parseQl(queryDataSource, metricInfo);
System.out.println(queryDataSource.ql);
Assert.assertTrue(StringUtils.equals(
"select count(1) as value from k8s_pod_mem_util where `period` <= 1691647903000 and `period` >= 1691644303000 and `app` in ('holoinsight-server','aaaaa') and `应用ID` = '111111111' group by `app` order by `period` acs",
queryDataSource.ql));

json =
"{\"start\":1691644303000,\"end\":1691647903000,\"name\":\"a\",\"metric\":\"k8s_pod_mem_util\",\"groupBy\":[\"app\",\"hostname\",\"workspace\",\"pod\",\"ip\",\"namespace\"],\"filters\":[{\"type\":\"literal_or\",\"name\":\"app\",\"value\":\"holoinsight-server|aaaaa\"},{\"type\":\"literal\",\"name\":\"应用ID\",\"value\":\"111111111\"},{\"type\":\"literal\",\"name\":\"fake应用ID\",\"value\":\"22222222\"}],\"select\":{\"app\":null,\"dd\":\"distinct(`aaa`)\"}}";
queryDataSource = J.fromJson(json, DataQueryRequest.QueryDataSource.class);
metricInfo = new MetricInfo();
metricInfo.setTags(J.toJson(Arrays.asList("app", "应用ID")));
queryFacade = new QueryFacadeImpl();
queryFacade.parseQl(queryDataSource, metricInfo);
System.out.println(queryDataSource.ql);
Assert.assertTrue(StringUtils.equals(
"select `app` , distinct(`aaa`) as `dd` from k8s_pod_mem_util where `period` <= 1691647903000 and `period` >= 1691644303000 and `app` in ('holoinsight-server','aaaaa') and `应用ID` = '111111111' group by `app` order by `period` acs",
queryDataSource.ql));
}
}

0 comments on commit 77704b0

Please sign in to comment.