From 33f0e4d614461f848b4cedbbcd33a541ab22ac12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=84=E9=B8=BF?= Date: Mon, 7 Aug 2023 17:17:57 +0800 Subject: [PATCH 1/3] feat(home): Log monitoring supports json logs --- .../server/home/dal/model/dto/conf/Rule.java | 6 ++ .../home/biz/common/GaeaSqlTaskUtil.java | 57 ++++++++++++++++++- .../server/home/biz/service/MetaService.java | 7 +++ .../home/biz/service/TenantInitService.java | 2 + .../impl/DefaultTenantInitServiceImpl.java | 6 ++ .../server/home/biz/ula/MonitorULA.java | 2 +- .../holoinsight/server/home/biz/ula/ULA.java | 2 +- .../server/home/biz/ula/ULAFacade.java | 5 +- .../home/task/impl/TenantAppMetaSyncTask.java | 1 + .../home/web/controller/MetaFacadeImpl.java | 5 ++ .../home/web/filter/Step3AuthFilter.java | 2 +- 11 files changed, 88 insertions(+), 7 deletions(-) diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/Rule.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/Rule.java index 092f25e76..042ca8e11 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/Rule.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/Rule.java @@ -57,4 +57,10 @@ public class Rule implements Serializable { * 默认值 */ public String defaultValue; + + /** ------------------JSON PATH 字段--------------------- */ + /** + * 表达式 + */ + public String jsonPathSyntax; } diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java index fa6bbd53c..edda352a0 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java @@ -61,6 +61,7 @@ public class GaeaSqlTaskUtil { private static final String leftRight = "LR"; private static final String separator = "SEP"; private static final String regexp = "REGEXP"; + private static final String json = "JSON"; private static final String dimColType = "DIM"; private static final String valColType = "VALUE"; @@ -155,6 +156,9 @@ public static From buildFrom(List logPaths, LogParse logParse, ExtraCon parse.setRegexp(new Log.Regexp()); parse.getRegexp().setExpression(logParse.regexp.expression); break; + case json: + parse.setType("json"); + break; default: parse.setType("none"); } @@ -225,6 +229,7 @@ public static From buildFrom(List logPaths, LogParse logParse, ExtraCon timeParse.setType("auto"); fromLog.setTime(timeParse); + boolean jsonTimeSelect = false; if (!CollectionUtils.isEmpty(splitCols)) { for (CustomPluginConf.SplitCol splitCol : splitCols) { if ("TIME".equals(splitCol.colType)) { @@ -248,10 +253,15 @@ public static From buildFrom(List logPaths, LogParse logParse, ExtraCon timeParse.setElect(buildElect(splitCol.rule, logParse.splitType)); timeParse.setType("elect"); timeParse.setFormat("golangLayout"); + jsonTimeSelect = true; } } fromLog.setTime(timeParse); } + if (logParse.splitType.equalsIgnoreCase(json) && !jsonTimeSelect) { + timeParse.setType("processTime"); + fromLog.setTime(timeParse); + } } From from = new From(); @@ -271,6 +281,7 @@ public static Where buildFrontFilterWhere(List whiteFilters, List whiteFilters, List whiteFilters, List whiteFilters, List { Where and = new Where(); Where not = new Where(); + Where.In in = new Where.In(); Elect elect = new Elect(); switch (w.getType()) { @@ -317,7 +339,6 @@ public static Where buildFrontFilterWhere(List whiteFilters, List whiteFilters, List getAppModelFromServerTable(String tenant, String serverTab if (CollectionUtils.isEmpty(mapList)) { return new ArrayList<>(); } + String[] s = StringUtils.split(serverTableName, "_"); List appModels = new ArrayList<>(); for (Map map : mapList) { @@ -72,6 +75,7 @@ public List getAppModelFromServerTable(String tenant, String serverTab } appModel.setApp(app); appModel.setWorkspace(workspace); + appModel.setTenant(map.getOrDefault(meta_tenant, s[0]).toString()); if (map.containsKey(meta_type) && null != map.get(meta_type)) { appModel.setMachineType(map.get(meta_type).toString()); @@ -90,6 +94,8 @@ public List getAppModelFromAppTable(String appTableName) { return appModels; } + String[] s = StringUtils.split(appTableName, "_"); + dbLists.forEach(db -> { if (null == db.get(meta_app)) { return; @@ -97,6 +103,7 @@ public List getAppModelFromAppTable(String appTableName) { AppModel appModel = new AppModel(); appModel.setApp(db.get(meta_app).toString()); appModel.setWorkspace(db.getOrDefault(meta_workspace, meta_workspace_default).toString()); + appModel.setTenant(db.getOrDefault(meta_tenant, s[0]).toString()); if (null == db.get("_label")) return; diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java index 68a229cb9..a814299fb 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java @@ -72,6 +72,8 @@ Boolean checkConditions(String tenant, String workspace, String metric, */ Map getTenantWorkspaceMetaConditions(String tenant, String workspace); + Map getTenantServerWorkspaceMetaConditions(String tenant, String workspace); + /** * add query filters by workspace * diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java index 91979a784..9dd209313 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java @@ -63,6 +63,12 @@ public Map getTenantWorkspaceMetaConditions(String tenant, Strin return new HashMap<>(); } + @Override + public Map getTenantServerWorkspaceMetaConditions(String tenant, + String workspace) { + return new HashMap<>(); + } + @Override public List getTenantFilters(String tenant, String workspace) { return new ArrayList<>(); diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/MonitorULA.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/MonitorULA.java index b47cf1556..da128d968 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/MonitorULA.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/MonitorULA.java @@ -102,7 +102,7 @@ public List getUserTenants(MonitorUser user) { } @Override - public MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms) { + public MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms) { String tenant = ms.getTenant(); if (null == tenant) { diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULA.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULA.java index 0e973288f..0680d9e28 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULA.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULA.java @@ -53,7 +53,7 @@ public interface ULA { * AUTH */ // 查看一个租户下的权限 - MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms); + MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms); // 是否是super用户 void checkSuper(MonitorUser user) throws Throwable; diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULAFacade.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULAFacade.java index 5e07dbfae..fb1fb1bac 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULAFacade.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/ula/ULAFacade.java @@ -116,8 +116,9 @@ public List getUsers(MonitorUser user, MonitorScope ms) { } // 取某个scope的最高权限 - public MonitorAuth getUserPowerPkg(MonitorUser user, MonitorScope ms) throws Throwable { - return getCurrentULA().getUserPowerPkg(user, ms); + public MonitorAuth getUserPowerPkg(HttpServletRequest req, MonitorUser user, MonitorScope ms) + throws Throwable { + return getCurrentULA().getUserPowerPkg(req, user, ms); } // 分析好cookie, 取回监控 user, 优先走本地校验 diff --git a/server/home/home-task/src/main/java/io/holoinsight/server/home/task/impl/TenantAppMetaSyncTask.java b/server/home/home-task/src/main/java/io/holoinsight/server/home/task/impl/TenantAppMetaSyncTask.java index d01198dae..d6d69fdc0 100644 --- a/server/home/home-task/src/main/java/io/holoinsight/server/home/task/impl/TenantAppMetaSyncTask.java +++ b/server/home/home-task/src/main/java/io/holoinsight/server/home/task/impl/TenantAppMetaSyncTask.java @@ -120,6 +120,7 @@ private void compare(String appTableName, List fromDbServers, List labelMap = new HashMap<>(); labelMap.put("machineType", appModel.getMachineType()); diff --git a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/MetaFacadeImpl.java b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/MetaFacadeImpl.java index 8e229ade1..c8315ed4b 100644 --- a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/MetaFacadeImpl.java +++ b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/MetaFacadeImpl.java @@ -100,6 +100,11 @@ public void doManage() { if (StringUtils.isNotBlank(ms.getWorkspace())) { queryExample.getParams().put("_workspace", ms.getWorkspace()); } + Map conditions = tenantInitService + .getTenantServerWorkspaceMetaConditions(ms.getTenant(), ms.getWorkspace()); + if (!CollectionUtils.isEmpty(conditions)) { + queryExample.getParams().putAll(conditions); + } List> list = dataClientService .queryByExample(tenantInitService.getTenantAppTable(ms.getTenant()), queryExample); diff --git a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/filter/Step3AuthFilter.java b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/filter/Step3AuthFilter.java index 5656e19a2..d4bd986a5 100644 --- a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/filter/Step3AuthFilter.java +++ b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/filter/Step3AuthFilter.java @@ -112,7 +112,7 @@ public boolean auth(HttpServletRequest req, HttpServletResponse resp) throws Thr return true; } } - ma = ulaFacade.getUserPowerPkg(mu, ms); + ma = ulaFacade.getUserPowerPkg(req, mu, ms); ulaFacade.checkWorkspace(req, mu, ms); if (null == ma || CollectionUtils.isEmpty(ma.powerConstants) || CollectionUtils.isEmpty(ma.getTenantViewPowerList())) { From 178c9091c3aabcc5f211d48a7d97db66f1d971ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=84=E9=B8=BF?= Date: Mon, 7 Aug 2023 20:26:48 +0800 Subject: [PATCH 2/3] add log-json-generated.py --- test/scenes/common/log-json-generator.py | 38 +++++++++++++++++++ test/scenes/scene-default/after.sh | 3 ++ test/scenes/scene-default/docker-compose.yaml | 1 + 3 files changed, 42 insertions(+) create mode 100644 test/scenes/common/log-json-generator.py diff --git a/test/scenes/common/log-json-generator.py b/test/scenes/common/log-json-generator.py new file mode 100644 index 000000000..90626ea4d --- /dev/null +++ b/test/scenes/common/log-json-generator.py @@ -0,0 +1,38 @@ +# coding=utf-8 +import atexit +import datetime +import sched +import time + +import logwriter + +s = sched.scheduler(time.time, time.sleep) + + +def schedule_with_fixed_rate(init_delay, interval, action, args): + def wrapper(*args2): + action() + now = datetime.datetime.now() + next_align_time = datetime.datetime.now().replace(microsecond=0) + datetime.timedelta(seconds=interval) + delta = next_align_time - now + s.enter(delta.total_seconds(), 1, wrapper, args2) + + s.enter(init_delay, 1, wrapper, args) + + +increase_value_log = logwriter.FileWrapper('test/json.log') +atexit.register(increase_value_log.close) + +start = int(time.time()) + + +def task1(*args): + time_str = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + cur = int(time.time()) + increase_value_log.write('{"time": "%s", "level":"%s", "biz":"biz1", "cost": "%dms"}\n' % (time_str, 'INFO', cur - start)) + increase_value_log.flush() + + +schedule_with_fixed_rate(0, 1, task1, ()) + +s.run() diff --git a/test/scenes/scene-default/after.sh b/test/scenes/scene-default/after.sh index a98dfacf2..bba72103c 100755 --- a/test/scenes/scene-default/after.sh +++ b/test/scenes/scene-default/after.sh @@ -26,3 +26,6 @@ docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c echo copy log-alert-generator.py to $server_container_name docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c ' python /home/admin/test/log-alert-generator.py & ' + +echo copy log-json-generator.py to $server_container_name +docker-compose exec -w /home/admin/logs/holoinsight-server -d -T server bash -c ' python /home/admin/test/log-json-generator.py & ' diff --git a/test/scenes/scene-default/docker-compose.yaml b/test/scenes/scene-default/docker-compose.yaml index 9fe095f41..543f541a5 100644 --- a/test/scenes/scene-default/docker-compose.yaml +++ b/test/scenes/scene-default/docker-compose.yaml @@ -84,6 +84,7 @@ services: - ../common/logwriter.py:/home/admin/test/logwriter.py:ro - ../common/log-generator.py:/home/admin/test/log-generator.py:ro - ../common/log-alert-generator.py:/home/admin/test/log-alert-generator.py:ro + - ../common/log-json-generator.py:/home/admin/test/log-json-generator.py:ro collector: extends: file: ../common/base.yaml From 075972093abfce1137c0cad78ca17f0df85e1389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=84=E9=B8=BF?= Date: Mon, 7 Aug 2023 20:35:33 +0800 Subject: [PATCH 3/3] fix: fix npe --- .../io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java index edda352a0..bbcfa20f1 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java @@ -258,7 +258,8 @@ public static From buildFrom(List logPaths, LogParse logParse, ExtraCon } fromLog.setTime(timeParse); } - if (logParse.splitType.equalsIgnoreCase(json) && !jsonTimeSelect) { + if (StringUtils.isNotBlank(logParse.splitType) && logParse.splitType.equalsIgnoreCase(json) + && !jsonTimeSelect) { timeParse.setType("processTime"); fromLog.setTime(timeParse); }