feat(home): Log monitoring supports json logs

server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/Rule.java

@@ -57,4 +57,10 @@ public class Rule implements Serializable {
    * 默认值
    */
   public String defaultValue;
+
+  /** ------------------JSON PATH 字段--------------------- */
+  /**
+   * 表达式
+   */
+  public String jsonPathSyntax;
 }

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java

@@ -61,6 +61,7 @@ public class GaeaSqlTaskUtil {
   private static final String leftRight = "LR";
   private static final String separator = "SEP";
   private static final String regexp = "REGEXP";
+  private static final String json = "JSON";
   private static final String dimColType = "DIM";
   private static final String valColType = "VALUE";
 
@@ -155,6 +156,9 @@ public static From buildFrom(List<LogPath> logPaths, LogParse logParse, ExtraCon
             parse.setRegexp(new Log.Regexp());
             parse.getRegexp().setExpression(logParse.regexp.expression);
             break;
+          case json:
+            parse.setType("json");
+            break;
           default:
             parse.setType("none");
         }
@@ -225,6 +229,7 @@ public static From buildFrom(List<LogPath> logPaths, LogParse logParse, ExtraCon
       timeParse.setType("auto");
       fromLog.setTime(timeParse);
 
+      boolean jsonTimeSelect = false;
       if (!CollectionUtils.isEmpty(splitCols)) {
         for (CustomPluginConf.SplitCol splitCol : splitCols) {
           if ("TIME".equals(splitCol.colType)) {
@@ -248,10 +253,16 @@ public static From buildFrom(List<LogPath> logPaths, LogParse logParse, ExtraCon
             timeParse.setElect(buildElect(splitCol.rule, logParse.splitType));
             timeParse.setType("elect");
             timeParse.setFormat("golangLayout");
+            jsonTimeSelect = true;
           }
         }
         fromLog.setTime(timeParse);
       }
+      if (StringUtils.isNotBlank(logParse.splitType) && logParse.splitType.equalsIgnoreCase(json)
+          && !jsonTimeSelect) {
+        timeParse.setType("processTime");
+        fromLog.setTime(timeParse);
+      }
     }
 
     From from = new From();
@@ -271,6 +282,7 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
 
         Where and = new Where();
         Elect elect = new Elect();
+        Where.In in = new Where.In();
         switch (w.type) {
           case leftRight:
             Elect.LeftRight leftRight = new Elect.LeftRight();
@@ -281,7 +293,6 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
             elect.setType("leftRight");
             elect.setLeftRight(leftRight);
 
-            Where.In in = new Where.In();
             in.setValues(w.values);
             in.setElect(elect);
             and.setIn(in);
@@ -295,6 +306,17 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
             and.setContainsAny(contains);
             break;
 
+          case json:
+            Elect.RefName refName = new Elect.RefName();
+            refName.setName(w.rule.jsonPathSyntax);
+            elect.setRefName(refName);
+            elect.setType("refName");
+
+            in.setValues(w.values);
+            in.setElect(elect);
+            and.setIn(in);
+            break;
+
           default:
             break;
         }
@@ -307,6 +329,7 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
       blackFilters.forEach(w -> {
         Where and = new Where();
         Where not = new Where();
+        Where.In in = new Where.In();
 
         Elect elect = new Elect();
         switch (w.getType()) {
@@ -317,7 +340,6 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
             leftRight.setRight(w.rule.right);
             elect.setType("leftRight");
             elect.setLeftRight(leftRight);
-            Where.In in = new Where.In();
             in.setValues(w.values);
             in.setElect(elect);
 
@@ -333,6 +355,18 @@ public static Where buildFrontFilterWhere(List<Filter> whiteFilters, List<Filter
             not.setContainsAny(contains);
             and.setNot(not);
             break;
+          case json:
+            Elect.RefName refName = new Elect.RefName();
+            refName.setName(w.rule.jsonPathSyntax);
+            elect.setType("refName");
+            elect.setRefName(refName);
+
+            in.setValues(w.values);
+            in.setElect(elect);
+
+            not.setIn(in);
+            and.setNot(not);
+            break;
           default:
             break;
         }
@@ -388,6 +422,11 @@ public static Where buildSampleWhere(String splitType,
             elect.getRefIndex().setIndex(rule.pos);
           }
           break;
+        case json:
+          elect.setType("refName");
+          elect.setRefName(new RefName());
+          elect.getRefName().setName(rule.jsonPathSyntax);
+          break;
         default:
           break;
       }
@@ -483,6 +522,11 @@ public static Where buildWhere(LogParse logParse, Map<String, Map<String, SplitC
               elect.getRefIndex().setIndex(rule.pos);
             }
             break;
+          case json:
+            elect.setType("refName");
+            elect.setRefName(new RefName());
+            elect.getRefName().setName(rule.jsonPathSyntax);
+            break;
           default:
             break;
         }
@@ -652,6 +696,11 @@ public static GroupBy buildGroupBy(LogParse logParse, ExtraConfig extraConfig,
             elect.getRefIndex().setIndex(rule.pos);
           }
           break;
+        case json:
+          elect.setType("refName");
+          elect.setRefName(new RefName());
+          elect.getRefName().setName(rule.jsonPathSyntax);
+          break;
         default:
           break;
       }
@@ -733,6 +782,11 @@ public static Elect buildElect(Rule rule, String splitType) {
           elect.getRefIndex().setIndex(rule.pos);
         }
         break;
+      case json:
+        elect.setType("refName");
+        elect.setRefName(new RefName());
+        elect.getRefName().setName(rule.jsonPathSyntax);
+        break;
       default:
         break;
     }

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/MetaService.java

@@ -11,6 +11,7 @@
 import io.holoinsight.server.home.common.util.StringUtil;
 import io.holoinsight.server.meta.facade.service.DataClientService;
 import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -35,6 +36,7 @@ public class MetaService {
   private static final String meta_type = "_type";
   private static final String MACHINE_TYPE = "machineType";
   private static final String meta_workspace = "_workspace";
+  private static final String meta_tenant = "tenant";
   private static final String meta_workspace_default = "default";
   @Autowired
   private DataClientService dataClientService;
@@ -52,6 +54,7 @@ public List<AppModel> getAppModelFromServerTable(String tenant, String serverTab
     if (CollectionUtils.isEmpty(mapList)) {
       return new ArrayList<>();
     }
+    String[] s = StringUtils.split(serverTableName, "_");
 
     List<AppModel> appModels = new ArrayList<>();
     for (Map<String, Object> map : mapList) {
@@ -72,6 +75,7 @@ public List<AppModel> getAppModelFromServerTable(String tenant, String serverTab
       }
       appModel.setApp(app);
       appModel.setWorkspace(workspace);
+      appModel.setTenant(map.getOrDefault(meta_tenant, s[0]).toString());
 
       if (map.containsKey(meta_type) && null != map.get(meta_type)) {
         appModel.setMachineType(map.get(meta_type).toString());
@@ -90,13 +94,16 @@ public List<AppModel> getAppModelFromAppTable(String appTableName) {
       return appModels;
     }
 
+    String[] s = StringUtils.split(appTableName, "_");
+
     dbLists.forEach(db -> {
       if (null == db.get(meta_app)) {
         return;
       }
       AppModel appModel = new AppModel();
       appModel.setApp(db.get(meta_app).toString());
       appModel.setWorkspace(db.getOrDefault(meta_workspace, meta_workspace_default).toString());
+      appModel.setTenant(db.getOrDefault(meta_tenant, s[0]).toString());
 
       if (null == db.get("_label"))
         return;

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java

@@ -72,6 +72,8 @@ Boolean checkConditions(String tenant, String workspace, String metric,
    */
   Map<String, String> getTenantWorkspaceMetaConditions(String tenant, String workspace);
 
+  Map<String, String> getTenantServerWorkspaceMetaConditions(String tenant, String workspace);
+
   /**
    * add query filters by workspace
    * 

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java

@@ -63,6 +63,12 @@ public Map<String, String> getTenantWorkspaceMetaConditions(String tenant, Strin
     return new HashMap<>();
   }
 
+  @Override
+  public Map<String, String> getTenantServerWorkspaceMetaConditions(String tenant,
+      String workspace) {
+    return new HashMap<>();
+  }
+
   @Override
   public List<QueryFilter> getTenantFilters(String tenant, String workspace) {
     return new ArrayList<>();

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/MonitorULA.java

@@ -102,7 +102,7 @@ public List<MonitorTenant> getUserTenants(MonitorUser user) {
   }
 
   @Override
-  public MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms) {
+  public MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms) {
 
     String tenant = ms.getTenant();
     if (null == tenant) {

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULA.java

@@ -53,7 +53,7 @@ public interface ULA {
    * AUTH
    */
   // 查看一个租户下的权限
-  MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms);
+  MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms);
 
   // 是否是super用户
   void checkSuper(MonitorUser user) throws Throwable;

server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULAFacade.java

@@ -116,8 +116,9 @@ public List<MonitorUser> getUsers(MonitorUser user, MonitorScope ms) {
   }
 
   // 取某个scope的最高权限
-  public MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms) throws Throwable {
-    return getCurrentULA().getUserPowerPkg(user, ms);
+  public MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms)
+      throws Throwable {
+    return getCurrentULA().getUserPowerPkg(req, user, ms);
   }
 
   // 分析好cookie, 取回监控 user, 优先走本地校验

server/home/home-task/src/main/java/io/holoinsight/server/home/task/impl/TenantAppMetaSyncTask.java

@@ -120,6 +120,7 @@ private void compare(String appTableName, List<AppModel> fromDbServers, List<App
       map.put("_type", meta_app);
       map.put("_workspace", appModel.getWorkspace());
       map.put("app", appModel.getApp());
+      map.put("tenant", appModel.getTenant());
 
       Map<String, Object> labelMap = new HashMap<>();
       labelMap.put("machineType", appModel.getMachineType());

server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/MetaFacadeImpl.java

@@ -100,6 +100,11 @@ public void doManage() {
         if (StringUtils.isNotBlank(ms.getWorkspace())) {
           queryExample.getParams().put("_workspace", ms.getWorkspace());
         }
+        Map<String, String> conditions = tenantInitService
+            .getTenantServerWorkspaceMetaConditions(ms.getTenant(), ms.getWorkspace());
+        if (!CollectionUtils.isEmpty(conditions)) {
+          queryExample.getParams().putAll(conditions);
+        }
 
         List<Map<String, Object>> list = dataClientService
             .queryByExample(tenantInitService.getTenantAppTable(ms.getTenant()), queryExample);

server/home/home-web/src/main/java/io/holoinsight/server/home/web/filter/Step3AuthFilter.java

@@ -112,7 +112,7 @@ public boolean auth(HttpServletRequest req, HttpServletResponse resp) throws Thr
           return true;
         }
       }
-      ma = ulaFacade.getUserPowerPkg(mu, ms);
+      ma = ulaFacade.getUserPowerPkg(req, mu, ms);
       ulaFacade.checkWorkspace(req, mu, ms);
       if (null == ma || CollectionUtils.isEmpty(ma.powerConstants)
           || CollectionUtils.isEmpty(ma.getTenantViewPowerList())) {

test/scenes/common/log-json-generator.py

@@ -0,0 +1,38 @@
+# coding=utf-8
+import atexit
+import datetime
+import sched
+import time
+
+import logwriter
+
+s = sched.scheduler(time.time, time.sleep)
+
+
+def schedule_with_fixed_rate(init_delay, interval, action, args):
+    def wrapper(*args2):
+        action()
+        now = datetime.datetime.now()
+        next_align_time = datetime.datetime.now().replace(microsecond=0) + datetime.timedelta(seconds=interval)
+        delta = next_align_time - now
+        s.enter(delta.total_seconds(), 1, wrapper, args2)
+
+    s.enter(init_delay, 1, wrapper, args)
+
+
+increase_value_log = logwriter.FileWrapper('test/json.log')
+atexit.register(increase_value_log.close)
+
+start = int(time.time())
+
+
+def task1(*args):
+    time_str = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    cur = int(time.time())
+    increase_value_log.write('{"time": "%s", "level":"%s", "biz":"biz1", "cost": "%dms"}\n' % (time_str, 'INFO', cur - start))
+    increase_value_log.flush()
+
+
+schedule_with_fixed_rate(0, 1, task1, ())
+
+s.run()

test/scenes/scene-default/after.sh

@@ -26,3 +26,6 @@ docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c
 
 echo copy log-alert-generator.py to $server_container_name
 docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c ' python /home/admin/test/log-alert-generator.py & '
+
+echo copy log-json-generator.py to $server_container_name
+docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c ' python /home/admin/test/log-json-generator.py & '

test/scenes/scene-default/docker-compose.yaml

@@ -84,6 +84,7 @@ services:
     - ../common/logwriter.py:/home/admin/test/logwriter.py:ro
     - ../common/log-generator.py:/home/admin/test/log-generator.py:ro
     - ../common/log-alert-generator.py:/home/admin/test/log-alert-generator.py:ro
+    - ../common/log-json-generator.py:/home/admin/test/log-json-generator.py:ro
   collector:
     extends:
       file: ../common/base.yaml