feat: ql query (#599)

traas-stack · Aug 11, 2023 · 427639d · 427639d
1 parent 4ba8141
commit 427639d
Show file tree

Hide file tree

Showing 6 changed files with 213 additions and 2 deletions.
diff --git a/...mon/common-service/src/main/java/io/holoinsight/server/common/model/DataQueryRequest.java b/...mon/common-service/src/main/java/io/holoinsight/server/common/model/DataQueryRequest.java
@@ -6,6 +6,7 @@
 import lombok.Data;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  *
@@ -55,6 +56,8 @@ public static class QueryDataSource {
     // instead of post-calculation
     public boolean apmMaterialized;
     public String ql;
+
+    public Map<String /* column name */, String /* expression */> select;
   }
 
   @Data

diff --git a/.../common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java b/.../common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java
@@ -4,6 +4,7 @@
 package io.holoinsight.server.common.service;
 
 import io.holoinsight.server.common.dao.entity.MetaDataDictValue;
+import io.holoinsight.server.common.dao.entity.MetricInfo;
 import io.holoinsight.server.query.grpc.QueryProto.QueryRequest;
 
 import java.util.Map;
@@ -17,4 +18,6 @@ public class SuperCache {
 
   public Map<String /* type */, Map<String /* k */, MetaDataDictValue>> metaDataDictValueMap;
   public Map<String, QueryRequest> expressionMetricList;
+
+  public Map<String /* metric table */, MetricInfo> metricInfoMap;
 }
diff --git a/.../common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java b/.../common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java
@@ -3,10 +3,20 @@
  */
 package io.holoinsight.server.common.service;
 
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import io.holoinsight.server.common.config.ProdLog;
 import io.holoinsight.server.common.config.ScheduleLoadTask;
+import io.holoinsight.server.common.dao.entity.MetricInfo;
+import io.holoinsight.server.common.dao.mapper.MetricInfoMapper;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.util.CollectionUtils;
+
+import javax.annotation.Resource;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  *
@@ -23,6 +33,9 @@ public class SuperCacheService extends ScheduleLoadTask {
   @Autowired
   private MetricInfoService metricInfoService;
 
+  @Resource
+  private MetricInfoMapper metricInfoMapper;
+
   public SuperCache getSc() {
     return sc;
   }
@@ -33,6 +46,7 @@ public void load() throws Exception {
     SuperCache sc = new SuperCache();
     sc.metaDataDictValueMap = metaDictValueService.getMetaDictValue();
     sc.expressionMetricList = metricInfoService.querySpmList();
+    sc.metricInfoMap = queryMetricInfoByPage();
     this.sc = sc;
     ProdLog.info("[SuperCache] load end");
   }
@@ -46,4 +60,30 @@ public int periodInSeconds() {
   public String getTaskName() {
     return "SuperCacheService";
   }
+
+
+  private Map<String, MetricInfo> queryMetricInfoByPage() {
+    QueryWrapper<MetricInfo> queryWrapper = new QueryWrapper<>();
+    queryWrapper.eq("deleted", 0);
+
+    int current = 1;
+    Map<String /* metric table */, MetricInfo> map = new HashMap<>();
+    int size = 1000;
+    do {
+      Page<MetricInfo> page = new Page<>(current++, size);
+      Page<MetricInfo> result = this.metricInfoMapper.selectPage(page, queryWrapper);
+
+      if (result == null || CollectionUtils.isEmpty(result.getRecords())) {
+        break;
+      }
+
+      for (MetricInfo metricInfo : result.getRecords()) {
+        if (StringUtils.isEmpty(metricInfo.getMetricTable())) {
+          continue;
+        }
+        map.put(metricInfo.getMetricTable(), metricInfo);
+      }
+    } while (current < 1000);
+    return map;
+  }
 }
diff --git a/...eresdbx/src/main/java/io/holoinsight/server/extension/ceresdbx/CeresdbxMetricStorage.java b/...eresdbx/src/main/java/io/holoinsight/server/extension/ceresdbx/CeresdbxMetricStorage.java
@@ -289,7 +289,7 @@ private Map<Map<String, String>, Result> getTagsWithResults(QueryParam queryPara
         if ("tsid".equals(name)) {
           continue;
         }
-        if ("timestamp".equals(name)) {
+        if ("timestamp".equals(name) || value.getDataType() == Value.DataType.Timestamp) {
           point.setTimestamp(value.getTimestamp());
           continue;
         }

diff --git a/...ome/home-web/src/main/java/io/holoinsight/server/home/web/controller/QueryFacadeImpl.java b/...ome/home-web/src/main/java/io/holoinsight/server/home/web/controller/QueryFacadeImpl.java
@@ -10,6 +10,8 @@
 import com.google.protobuf.util.JsonFormat;
 import io.holoinsight.server.common.LatchWork;
 import io.holoinsight.server.common.UtilMisc;
+import io.holoinsight.server.common.dao.entity.MetricInfo;
+import io.holoinsight.server.common.service.SuperCacheService;
 import io.holoinsight.server.common.threadpool.CommonThreadPools;
 import io.holoinsight.server.home.biz.common.MetaDictUtil;
 import io.holoinsight.server.home.biz.service.TenantInitService;
@@ -59,10 +61,13 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -84,6 +89,9 @@ public class QueryFacadeImpl extends BaseFacade {
   @Autowired
   private TenantInitService tenantInitService;
 
+  @Autowired
+  private SuperCacheService superCacheService;
+
   @PostMapping
   public JsonResult<QueryResponse> query(@RequestBody DataQueryRequest request) {
 
@@ -471,9 +479,15 @@ public QueryProto.QueryRequest convertRequest(DataQueryRequest request) {
       if ((System.currentTimeMillis() - d.start) < 80000L) {
         d.start -= 60000L;
       }
+      MetricInfo metricInfo = getMetricInfo(d.getMetric());
+      if (isCeresdb4Storage(metricInfo)) {
+        parseQl(d, metricInfo);
+      }
 
       QueryProto.Datasource.Builder datasourceBuilder = QueryProto.Datasource.newBuilder();
-      toProtoBean(datasourceBuilder, d);
+      Map<String, Object> objMap = J.toMap(J.toJson(d));
+      objMap.remove("select");
+      toProtoBean(datasourceBuilder, objMap);
       Boolean aBoolean = tenantInitService.checkConditions(ms.getTenant(), ms.getWorkspace(),
           datasourceBuilder.getMetric(), datasourceBuilder.getFiltersList());
       if (!aBoolean) {
@@ -487,6 +501,107 @@ public QueryProto.QueryRequest convertRequest(DataQueryRequest request) {
     return builder.build();
   }
 
+
+
+  protected void parseQl(QueryDataSource d, MetricInfo metricInfo) {
+    Set<String> tags = new HashSet<>(J.toList(metricInfo.getTags()));
+    StringBuilder select = new StringBuilder("select ");
+    List<String> selects = new ArrayList<>();
+    if (CollectionUtils.isEmpty(d.select)) {
+      selects.add(" count(1) as value ");
+    } else {
+      for (Map.Entry<String /* column name */, String /* expression */> entry : d.select
+          .entrySet()) {
+        String columnName = entry.getKey();
+        String expression = entry.getValue();
+        if (StringUtils.isEmpty(expression)) {
+          selects.add("`" + columnName + "`");
+        } else {
+          selects.add(expression + " as " + columnName);
+        }
+      }
+    }
+    selects.add("`period`");
+    select.append(String.join(" , ", selects));
+    select.append(" from ").append(d.metric);
+    select.append(" where `period` <= ") //
+        .append(d.end) //
+        .append(" and `period` >= ") //
+        .append(d.start);
+    if (!CollectionUtils.isEmpty(d.filters)) {
+      for (QueryFilter filter : d.filters) {
+        if (!tags.contains(filter.name)) {
+          continue;
+        }
+        switch (filter.type) {
+          case "literal_or":
+            select.append(" and `").append(filter.name).append("` in ('")
+                .append(String.join("','", Arrays.asList(filter.value.split("\\|")))).append("')");
+            break;
+          case "not_literal_or":
+            select.append(" and ").append(filter.name).append(" not in ('")
+                .append(String.join("','", Arrays.asList(filter.value.split("\\|")))).append("')");
+            break;
+          case "wildcard":
+            select.append(" and `").append(filter.name).append("` like '").append(filter.value)
+                .append("'");
+            break;
+          case "regexp":
+            select.append(" and `").append(filter.name).append("` REGEXP '").append(filter.value)
+                .append("'");
+            break;
+          case "not_regexp_match":
+            select.append(" and `").append(filter.name).append("` NOT REGEXP '")
+                .append(filter.value).append("'");
+            break;
+          case "literal":
+            select.append(" and `").append(filter.name).append("` = '").append(filter.value)
+                .append("'");
+            break;
+          case "not_literal":
+            select.append(" and `").append(filter.name).append("` <> '").append(filter.value)
+                .append("'");
+            break;
+        }
+      }
+    }
+    List<String> gbList = new ArrayList<>();
+    gbList.add("period");
+    if (!CollectionUtils.isEmpty(d.groupBy)) {
+      for (String gb : d.groupBy) {
+        if (!tags.contains(gb)) {
+          continue;
+        }
+        gbList.add(gb);
+      }
+    }
+    select.append(" group by `")//
+        .append(String.join("` , `", gbList)) //
+        .append("`");
+    select.append(" order by `period` asc");
+    log.info("parse sql {}", select);
+    d.setQl(select.toString());
+  }
+
+  private MetricInfo getMetricInfo(String metric) {
+    if (CollectionUtils.isEmpty(this.superCacheService.getSc().metricInfoMap)
+        || StringUtils.isEmpty(metric)) {
+      return null;
+    }
+    return this.superCacheService.getSc().metricInfoMap.get(metric);
+  }
+
+  private boolean isCeresdb4Storage(MetricInfo metricInfo) {
+    if (metricInfo == null) {
+      return false;
+    }
+    if (StringUtils.isNotEmpty(metricInfo.getStorageTenant())
+        && StringUtils.equals(metricInfo.getStorageTenant(), "ceresdb4")) {
+      return true;
+    }
+    return false;
+  }
+
   private long getInterval(String downsample) {
     long interval = 60000L;
     switch (downsample) {

diff --git a/...home-web/src/test/java/io/holoinsight/server/home/web/controller/QueryFacadeImplTest.java b/...home-web/src/test/java/io/holoinsight/server/home/web/controller/QueryFacadeImplTest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+package io.holoinsight.server.home.web.controller;
+
+import io.holoinsight.server.common.J;
+import io.holoinsight.server.common.dao.entity.MetricInfo;
+import io.holoinsight.server.common.model.DataQueryRequest;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author masaimu
+ * @version 2023-08-10 14:56:00
+ */
+public class QueryFacadeImplTest {
+
+  @Test
+  public void name() {
+    String json =
+        "{\"start\":1691644303000,\"end\":1691647903000,\"name\":\"a\",\"metric\":\"k8s_pod_mem_util\",\"groupBy\":[\"app\",\"hostname\",\"workspace\",\"pod\",\"ip\",\"namespace\"],\"filters\":[{\"type\":\"literal_or\",\"name\":\"app\",\"value\":\"holoinsight-server|aaaaa\"},{\"type\":\"literal\",\"name\":\"应用ID\",\"value\":\"111111111\"},{\"type\":\"literal\",\"name\":\"fake应用ID\",\"value\":\"22222222\"}]}";
+    DataQueryRequest.QueryDataSource queryDataSource =
+        J.fromJson(json, DataQueryRequest.QueryDataSource.class);
+    MetricInfo metricInfo = new MetricInfo();
+    metricInfo.setTags(J.toJson(Arrays.asList("app", "应用ID")));
+    QueryFacadeImpl queryFacade = new QueryFacadeImpl();
+    queryFacade.parseQl(queryDataSource, metricInfo);
+    System.out.println(queryDataSource.ql);
+    Assert.assertTrue(StringUtils.equals(
+        "select  count(1) as value  , `period` from k8s_pod_mem_util where `period` <= 1691647903000 and `period` >= 1691644303000 and `app` in ('holoinsight-server','aaaaa') and `应用ID` = '111111111' group by `period` , `app` order by `period` asc",
+        queryDataSource.ql));
+
+    json =
+        "{\"start\":1691644303000,\"end\":1691647903000,\"name\":\"a\",\"metric\":\"k8s_pod_mem_util\",\"groupBy\":[\"app\",\"hostname\",\"workspace\",\"pod\",\"ip\",\"namespace\"],\"filters\":[{\"type\":\"literal_or\",\"name\":\"app\",\"value\":\"holoinsight-server|aaaaa\"},{\"type\":\"literal\",\"name\":\"应用ID\",\"value\":\"111111111\"},{\"type\":\"literal\",\"name\":\"fake应用ID\",\"value\":\"22222222\"}],\"select\":{\"app\":null,\"dd\":\"distinct(`aaa`)\"}}";
+    queryDataSource = J.fromJson(json, DataQueryRequest.QueryDataSource.class);
+    metricInfo = new MetricInfo();
+    metricInfo.setTags(J.toJson(Arrays.asList("app", "应用ID")));
+    queryFacade = new QueryFacadeImpl();
+    queryFacade.parseQl(queryDataSource, metricInfo);
+    System.out.println(queryDataSource.ql);
+    Assert.assertTrue(StringUtils.equals(
+        "select `app` , distinct(`aaa`) as dd , `period` from k8s_pod_mem_util where `period` <= 1691647903000 and `period` >= 1691644303000 and `app` in ('holoinsight-server','aaaaa') and `应用ID` = '111111111' group by `period` , `app` order by `period` asc",
+        queryDataSource.ql));
+  }
+}