feat(home): support query by promql (#854)

traas-stack · May 22, 2024 · adec9f4 · adec9f4
1 parent 3370b6d
commit adec9f4
Show file tree

Hide file tree

Showing 32 changed files with 8,147 additions and 3,275 deletions.
diff --git a/deploy/examples/docker-compose/.env b/deploy/examples/docker-compose/.env
@@ -1,4 +1,4 @@
 mysql_image=mysql:8
-ceresdb_image=ceresdb/ceresdb-server:v1.0.0
+ceresdb_image=ceresdb/ceresdb-server:latest
 server_image=holoinsight/server:latest
 agent_image=holoinsight/agent:latest
diff --git a/...rver/home/task/MetricCrawlerConstant.java → ...r/common/model/MetricCrawlerConstant.java b/...rver/home/task/MetricCrawlerConstant.java → ...r/common/model/MetricCrawlerConstant.java
@@ -2,7 +2,7 @@
  * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
  */
 
-package io.holoinsight.server.home.task;
+package io.holoinsight.server.common.model;
 
 /**
  * @author jsy1001de
@@ -14,7 +14,12 @@ public class MetricCrawlerConstant {
   public static final String GLOBAL_WORKSPACE = "-";
   public static final String GLOBAL_ORGANIZATION = "-";
   public static final String PERCENT_UNIT = "percent";
+  public static final String RATE_UNIT = "rate";
+
   public static final String NUMBER_UNIT = "number";
   public static final String BYTES_UNIT = "bytes";
   public static final String BITS_UNIT = "bits";
+  public static final String MS_UNIT = "millisecond";
+  public static final String SEC_UNIT = "second";
+  public static final String US_UNIT = "microsecond";
 }
diff --git a/...eresdbx/src/main/java/io/holoinsight/server/extension/ceresdbx/CeresdbxMetricStorage.java b/...eresdbx/src/main/java/io/holoinsight/server/extension/ceresdbx/CeresdbxMetricStorage.java
@@ -22,6 +22,7 @@
 import io.holoinsight.server.common.J;
 import io.holoinsight.server.extension.MetricMeterService;
 import io.holoinsight.server.extension.model.DetailResult;
+import io.holoinsight.server.extension.model.PqlLabelParam;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -226,6 +227,33 @@ public List<Result> pqlRangeQuery(PqlParam pqlParam) {
     return pqlQueryService.queryRange(pqlParam);
   }
 
+  @Override
+  public List<Map<String, String>> pqlSeriesQuery(PqlLabelParam pqlLabelParam) {
+    if (Objects.isNull(pqlQueryService)) {
+      LOGGER.warn("[CeresDB] pqlSeriesQuery is null");
+      return Lists.newArrayList();
+    }
+    return pqlQueryService.querySeries(pqlLabelParam);
+  }
+
+  @Override
+  public List<String> pqlLabelsQuery(PqlLabelParam pqlLabelParam) {
+    if (Objects.isNull(pqlQueryService)) {
+      LOGGER.warn("[CeresDB] pqlLabelsQuery is null");
+      return Lists.newArrayList();
+    }
+    return pqlQueryService.queryLabels(pqlLabelParam);
+  }
+
+  @Override
+  public List<String> pqlLabelValueQuery(PqlLabelParam pqlLabelParam) {
+    if (Objects.isNull(pqlQueryService)) {
+      LOGGER.warn("[CeresDB] pqlLabelValueQuery is null");
+      return Lists.newArrayList();
+    }
+    return pqlQueryService.queryLabelValues(pqlLabelParam);
+  }
+
   @Override
   public DetailResult queryDetail(QueryParam queryParam) {
     String sql = queryParam.getQl();

diff --git a/...mote-promql/src/main/java/io/holoinsight/server/extension/promql/HttpPqlQueryService.java b/...mote-promql/src/main/java/io/holoinsight/server/extension/promql/HttpPqlQueryService.java
@@ -14,10 +14,12 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
+import io.holoinsight.server.extension.model.PqlLabelParam;
 import io.holoinsight.server.extension.model.PqlParam;
 import io.holoinsight.server.extension.model.QueryResult;
 import io.holoinsight.server.extension.model.QueryResult.Result;
 import io.holoinsight.server.extension.promql.model.Endpoint;
+import io.holoinsight.server.extension.promql.model.PqlLabelResult;
 import io.holoinsight.server.extension.promql.model.PqlResult;
 import io.holoinsight.server.extension.promql.model.PqlResult.Data;
 import io.holoinsight.server.extension.promql.utils.HttpClientUtils;
@@ -40,6 +42,9 @@ public class HttpPqlQueryService implements PqlQueryService {
   public static final String SUCCESS = "success";
   public static final String DEFAULT_TENANT = "default";
   public static final String API_V1_QUERY_RANGE = "api/v1/query_range";
+  public static final String API_V1_QUERY_SERIES = "api/v1/series";
+  public static final String API_V1_QUERY_LABELS = "api/v1/labels";
+  public static final String API_V1_QUERY_LABEL_VALUES = "api/v1/label/%s/values";
   public static final String API_V1_QUERY = "api/v1/query";
   private Map<String, Endpoint> endpoints;
 
@@ -66,13 +71,8 @@ private Endpoint getEndpoint(String tenant) {
 
   private Map<String, Object> parasParam(PqlParam pqlParam) {
     HashMap<String, Object> params = Maps.newHashMap();
-    try {
-      params.put("query", URLEncoder.encode(pqlParam.getQuery(), "UTF-8"));
-    } catch (UnsupportedEncodingException e) {
-      LOGGER.error("[PqlRemoteQuery] encode pql query param error , pql : {}", pqlParam.getQuery(),
-          e);
-      throw new RuntimeException("encode pql query param error");
-    }
+
+    params.put("query", pqlParam.getQuery());
     long time = pqlParam.getTime();
     if (time > 0) {
       params.put("time", unixTimestamp(time));
@@ -83,7 +83,7 @@ private Map<String, Object> parasParam(PqlParam pqlParam) {
       params.put("timeout", Integer.parseInt(timeout));
     }
     if (pqlParam.getStep() > 0) {
-      params.put("step", pqlParam.getStep());
+      params.put("step", pqlParam.getStep() / 1000);
     }
     if (pqlParam.getStart() > 0) {
       params.put("start", unixTimestamp(pqlParam.getStart()));
@@ -153,11 +153,11 @@ private QueryResult.Point getPoint(List value) {
     return point;
   }
 
-  private static double unixTimestamp(long ms) {
+  private static long unixTimestamp(long ms) {
     if (ms == 0) {
       ms = System.currentTimeMillis();
     }
-    return (double) ms / 1000;
+    return ms / 1000;
   }
 
   @Override
@@ -172,6 +172,96 @@ public List<Result> queryRange(PqlParam pqlParam) {
     return toResult(pqlParam, body);
   }
 
+  @Override
+  public List<Map<String, String>> querySeries(PqlLabelParam pqlLabelParam) {
+    String tenant = pqlLabelParam.getTenant();
+    Endpoint endpoint = getEndpoint(tenant);
+    if (Objects.isNull(endpoint)) {
+      LOGGER.error("[PqlRemoteQuery] queryRange error, tenant {} endpoint is not exists", tenant);
+      return Lists.newArrayList();
+    }
+    String body =
+        HttpClientUtils.get(endpoint, API_V1_QUERY_SERIES, parasLabelParam(pqlLabelParam));
+    List<Map<String, String>> results = Lists.newArrayList();
+    if (StringUtils.isBlank(body)) {
+      return results;
+    }
+    PqlLabelResult<List<Map<String, String>>> pqlResult =
+        new Gson().fromJson(body, PqlLabelResult.class);
+    if (!StringUtils.equalsIgnoreCase(SUCCESS, pqlResult.getStatus())) {
+      LOGGER.error("[PqlRemoteQuery] exec pql series: {} error, body:{}", pqlLabelParam, body);
+      return results;
+    }
+
+    return pqlResult.getData();
+  }
+
+  @Override
+  public List<String> queryLabels(PqlLabelParam pqlLabelParam) {
+    String tenant = pqlLabelParam.getTenant();
+    Endpoint endpoint = getEndpoint(tenant);
+    if (Objects.isNull(endpoint)) {
+      LOGGER.error("[PqlRemoteQuery] queryRange error, tenant {} endpoint is not exists", tenant);
+      return Lists.newArrayList();
+    }
+    String body =
+        HttpClientUtils.get(endpoint, API_V1_QUERY_LABELS, parasLabelParam(pqlLabelParam));
+    List<String> results = Lists.newArrayList();
+    if (StringUtils.isBlank(body)) {
+      return results;
+    }
+    PqlLabelResult<List<String>> pqlResult = new Gson().fromJson(body, PqlLabelResult.class);
+    if (!StringUtils.equalsIgnoreCase(SUCCESS, pqlResult.getStatus())) {
+      LOGGER.error("[PqlRemoteQuery] exec pql series: {} error, body:{}", pqlLabelParam, body);
+      return results;
+    }
+
+    return pqlResult.getData();
+  }
+
+  @Override
+  public List<String> queryLabelValues(PqlLabelParam pqlLabelParam) {
+    String tenant = pqlLabelParam.getTenant();
+    Endpoint endpoint = getEndpoint(tenant);
+    if (Objects.isNull(endpoint)) {
+      LOGGER.error("[PqlRemoteQuery] queryRange error, tenant {} endpoint is not exists", tenant);
+      return Lists.newArrayList();
+    }
+    String body = HttpClientUtils.get(endpoint,
+        String.format(API_V1_QUERY_LABEL_VALUES, pqlLabelParam.getLabelName()),
+        parasLabelParam(pqlLabelParam));
+    List<String> results = Lists.newArrayList();
+    if (StringUtils.isBlank(body)) {
+      return results;
+    }
+    PqlLabelResult<List<String>> pqlResult = new Gson().fromJson(body, PqlLabelResult.class);
+    if (!StringUtils.equalsIgnoreCase(SUCCESS, pqlResult.getStatus())) {
+      LOGGER.error("[PqlRemoteQuery] exec pql series: {} error, body:{}", pqlLabelParam, body);
+      return results;
+    }
+
+    return pqlResult.getData();
+  }
+
+
+  private Map<String, Object> parasLabelParam(PqlLabelParam pqlLabelParam) {
+    HashMap<String, Object> params = Maps.newHashMap();
+
+    if (pqlLabelParam.getStart() > 0) {
+      params.put("start", unixTimestamp(pqlLabelParam.getStart()));
+    }
+    if (pqlLabelParam.getEnd() > 0) {
+      params.put("end", unixTimestamp(pqlLabelParam.getEnd()));
+    }
+
+    if (null != pqlLabelParam.getMatch()) {
+      for (String ma : pqlLabelParam.getMatch()) {
+        params.put("match[]", ma);
+      }
+    }
+    return params;
+  }
+
   public void addEndpoints(String tenant, Endpoint endpoint) {
     this.endpoints.put(tenant, endpoint);
   }

diff --git a/...e-remote-promql/src/main/java/io/holoinsight/server/extension/promql/PqlQueryService.java b/...e-remote-promql/src/main/java/io/holoinsight/server/extension/promql/PqlQueryService.java
@@ -4,7 +4,9 @@
 package io.holoinsight.server.extension.promql;
 
 import java.util.List;
+import java.util.Map;
 
+import io.holoinsight.server.extension.model.PqlLabelParam;
 import io.holoinsight.server.extension.model.PqlParam;
 import io.holoinsight.server.extension.model.QueryResult.Result;
 
@@ -32,4 +34,10 @@ public interface PqlQueryService {
    * @return
    */
   List<Result> queryRange(PqlParam pqlParam);
+
+  List<Map<String, String>> querySeries(PqlLabelParam pqlLabelParam);
+
+  List<String> queryLabels(PqlLabelParam pqlLabelParam);
+
+  List<String> queryLabelValues(PqlLabelParam pqlLabelParam);
 }
diff --git a/...ote-promql/src/main/java/io/holoinsight/server/extension/promql/model/PqlLabelResult.java b/...ote-promql/src/main/java/io/holoinsight/server/extension/promql/model/PqlLabelResult.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+
+package io.holoinsight.server.extension.promql.model;
+
+import lombok.Data;
+
+/**
+ * @author jsy1001de
+ * @version 1.0: PqlLabelResult.java, Date: 2024-05-21 Time: 21:06
+ */
+@Data
+public class PqlLabelResult<T> {
+  private String status;
+
+  private T data;
+
+}
diff --git a/...ension/extension-storage/src/main/java/io/holoinsight/server/extension/MetricStorage.java b/...ension/extension-storage/src/main/java/io/holoinsight/server/extension/MetricStorage.java
@@ -4,8 +4,10 @@
 package io.holoinsight.server.extension;
 
 import java.util.List;
+import java.util.Map;
 
 import io.holoinsight.server.extension.model.DetailResult;
+import io.holoinsight.server.extension.model.PqlLabelParam;
 import io.holoinsight.server.extension.model.QueryMetricsParam;
 import io.holoinsight.server.extension.model.QueryResult.Result;
 import io.holoinsight.server.extension.model.WriteMetricsParam;
@@ -35,6 +37,12 @@ public interface MetricStorage {
 
   List<Result> pqlRangeQuery(PqlParam pqlParam);
 
+  List<Map<String, String>> pqlSeriesQuery(PqlLabelParam pqlLabelParam);
+
+  List<String> pqlLabelsQuery(PqlLabelParam pqlLabelParam);
+
+  List<String> pqlLabelValueQuery(PqlLabelParam pqlLabelParam);
+
   DetailResult queryDetail(QueryParam queryParam);
 
 }
diff --git a/...on/extension-storage/src/main/java/io/holoinsight/server/extension/NoopMetricStorage.java b/...on/extension-storage/src/main/java/io/holoinsight/server/extension/NoopMetricStorage.java
@@ -5,12 +5,15 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import io.holoinsight.server.extension.model.DetailResult;
+import io.holoinsight.server.extension.model.PqlLabelParam;
 import io.holoinsight.server.extension.model.PqlParam;
 import io.holoinsight.server.extension.model.QueryMetricsParam;
 import io.holoinsight.server.extension.model.QueryParam;
 import io.holoinsight.server.extension.model.QueryResult;
+import io.holoinsight.server.extension.model.QueryResult.Result;
 import io.holoinsight.server.extension.model.WriteMetricsParam;
 import reactor.core.publisher.Mono;
 
@@ -61,6 +64,21 @@ public List<QueryResult.Result> pqlRangeQuery(PqlParam pqlParam) {
     return Collections.emptyList();
   }
 
+  @Override
+  public List<Map<String, String>> pqlSeriesQuery(PqlLabelParam pqlLabelParam) {
+    return null;
+  }
+
+  @Override
+  public List<String> pqlLabelsQuery(PqlLabelParam pqlLabelParam) {
+    return null;
+  }
+
+  @Override
+  public List<String> pqlLabelValueQuery(PqlLabelParam pqlLabelParam) {
+    return null;
+  }
+
   @Override
   public DetailResult queryDetail(QueryParam queryParam) {
     return DetailResult.empty();

diff --git a/.../extension-storage/src/main/java/io/holoinsight/server/extension/model/PqlLabelParam.java b/.../extension-storage/src/main/java/io/holoinsight/server/extension/model/PqlLabelParam.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+
+package io.holoinsight.server.extension.model;
+
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author jsy1001de
+ * @version 1.0: PqlLabelParam.java, Date: 2024-05-21 Time: 20:28
+ */
+@Data
+public class PqlLabelParam {
+  String tenant;
+  String metric;
+  String labelName;
+  List<String> match;
+  long start;
+  long end;
+}