From 5783c8202ba1f1317dc856bfbdf06feb2c455f03 Mon Sep 17 00:00:00 2001 From: "xiangfeng.xzc" Date: Wed, 28 Feb 2024 11:27:28 +0800 Subject: [PATCH] feat: loganalysis agg --- server/agg/agg-core/pom.xml | 4 + .../server/agg/v1/core/conf/OutputField.java | 2 + .../AggDispatcherAutoConfiguration.java | 1 + .../AggDispatcherMockDataGenerator.java | 66 ------- .../mock/AggDispatcherMockDataGenerator.java | 176 ++++++++++++++++++ .../agg/v1/dispatcher/mock/AnalyzedLog.java | 22 +++ .../server/agg/v1/dispatcher/mock/LAPart.java | 29 +++ .../agg/v1/dispatcher/mock/SourceWord.java | 22 +++ .../v1/executor/executor/AggTaskExecutor.java | 6 + .../agg/v1/executor/executor/AnalyzedLog.java | 143 ++++++++++++++ .../agg/v1/executor/executor/GroupField.java | 15 ++ .../agg/v1/executor/executor/KnownValue.java | 38 ++++ .../agg/v1/executor/executor/LAPart.java | 59 ++++++ .../agg/v1/executor/executor/LogAnalysis.java | 19 ++ .../executor/executor/LogAnalysisState.java | 84 +++++++++ .../agg/v1/executor/executor/SourceWord.java | 18 ++ .../v1/executor/executor/UnknownValue.java | 50 +++++ .../query/service/analysis/AggCenter.java | 29 ++- .../query/service/analysis/Analysis.java | 16 +- 19 files changed, 722 insertions(+), 77 deletions(-) delete mode 100644 server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherMockDataGenerator.java create mode 100644 server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AggDispatcherMockDataGenerator.java create mode 100644 server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AnalyzedLog.java create mode 100644 server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/LAPart.java create mode 100644 server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/SourceWord.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AnalyzedLog.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/KnownValue.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LAPart.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysis.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysisState.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/SourceWord.java create mode 100644 server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/UnknownValue.java diff --git a/server/agg/agg-core/pom.xml b/server/agg/agg-core/pom.xml index 0ff2ac3ec..613e4aa59 100644 --- a/server/agg/agg-core/pom.xml +++ b/server/agg/agg-core/pom.xml @@ -61,6 +61,10 @@ com.alibaba fastjson + + com.fasterxml.jackson.core + jackson-annotations + diff --git a/server/agg/agg-core/src/main/java/io/holoinsight/server/agg/v1/core/conf/OutputField.java b/server/agg/agg-core/src/main/java/io/holoinsight/server/agg/v1/core/conf/OutputField.java index 2aae011fe..28e0e1b81 100644 --- a/server/agg/agg-core/src/main/java/io/holoinsight/server/agg/v1/core/conf/OutputField.java +++ b/server/agg/agg-core/src/main/java/io/holoinsight/server/agg/v1/core/conf/OutputField.java @@ -5,6 +5,7 @@ import javax.annotation.Nonnull; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.Expression; @@ -58,6 +59,7 @@ public static OutputField create(String name, String expr) { return new OutputField(name, expr); } + @JsonIgnore public Expression getCompiledExpression() { if (compiledExpression == null) { // TODO compile error? diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherAutoConfiguration.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherAutoConfiguration.java index 2af1260aa..a0f6f21db 100644 --- a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherAutoConfiguration.java +++ b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherAutoConfiguration.java @@ -11,6 +11,7 @@ import org.springframework.context.annotation.Import; import io.holoinsight.server.agg.v1.core.AggProperties; +import io.holoinsight.server.agg.v1.dispatcher.mock.AggDispatcherMockDataGenerator; import io.holoinsight.server.common.dao.CommonDaoConfiguration; import io.holoinsight.server.common.springboot.ConditionalOnRole; diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherMockDataGenerator.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherMockDataGenerator.java deleted file mode 100644 index 2f5888cab..000000000 --- a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherMockDataGenerator.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. - */ -package io.holoinsight.server.agg.v1.dispatcher; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Random; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; - -import io.holoinsight.server.common.auth.AuthInfo; -import io.holoinsight.server.extension.model.Header; -import io.holoinsight.server.extension.model.Row; -import io.holoinsight.server.extension.model.Table; -import lombok.extern.slf4j.Slf4j; - -/** - *

- * created at 2023/10/17 - * - * @author xzchaoo - */ -@Slf4j -public class AggDispatcherMockDataGenerator { - @Autowired - private AggDispatcher aggDispatcher; - - @Scheduled(initialDelay = 1000L, fixedDelay = 1000L) - public void execute() { - AuthInfo ai = new AuthInfo(); - ai.setTenant("monitor"); - - long ts = System.currentTimeMillis() / 1000 * 1000; - - Random r = new Random(); - - Table table = new Table(); - - Header header = new Header(); - header.setTagKeys(Arrays.asList("app", "userId")); - header.setFieldKeys(Arrays.asList("count", "count1")); - table.setHeader(header); - - table.setRows(new ArrayList<>(100)); - - table.setName("test"); - table.setTimestamp(ts); - - for (int i = 0; i < 100; i++) { - Row row = new Row(); - row.setTimestamp(ts); - - String app = "foo" + (r.nextInt(5)); - String userId = "user" + (r.nextInt(100)); - row.setTagValues(Arrays.asList(app, userId)); - - row.setFieldValues(Arrays.asList(1D, 1D)); - - table.getRows().add(row); - } - - aggDispatcher.dispatchDetailData(ai, table); - } -} diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AggDispatcherMockDataGenerator.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AggDispatcherMockDataGenerator.java new file mode 100644 index 000000000..6e0ef6d6e --- /dev/null +++ b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AggDispatcherMockDataGenerator.java @@ -0,0 +1,176 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.dispatcher.mock; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; + +import io.holoinsight.server.agg.v1.dispatcher.AggDispatcher; +import io.holoinsight.server.common.JsonUtils; +import io.holoinsight.server.common.auth.AuthInfo; +import io.holoinsight.server.extension.model.Header; +import io.holoinsight.server.extension.model.Record; +import io.holoinsight.server.extension.model.Row; +import io.holoinsight.server.extension.model.Table; +import lombok.extern.slf4j.Slf4j; + +/** + *

+ * created at 2023/10/17 + * + * @author xzchaoo + */ +@Slf4j +public class AggDispatcherMockDataGenerator { + @Autowired + private AggDispatcher aggDispatcher; + + @Scheduled(initialDelay = 1000L, fixedDelay = 1000L) + public void execute() { + AuthInfo ai = new AuthInfo(); + ai.setTenant("monitor"); + + long ts = System.currentTimeMillis() / 1000 * 1000; + + Random r = new Random(); + + Table table = new Table(); + + Header header = new Header(); + header.setTagKeys(Arrays.asList("app", "userId")); + header.setFieldKeys(Arrays.asList("count", "count1")); + table.setHeader(header); + + table.setRows(new ArrayList<>(100)); + + table.setName("test"); + table.setTimestamp(ts); + + for (int i = 0; i < 100; i++) { + Row row = new Row(); + row.setTimestamp(ts); + + String app = "foo" + (r.nextInt(5)); + String userId = "user" + (r.nextInt(100)); + row.setTagValues(Arrays.asList(app, userId)); + + row.setFieldValues(Arrays.asList(1D, 1D)); + + table.getRows().add(row); + } + + aggDispatcher.dispatchDetailData(ai, table); + } + + @Scheduled(initialDelay = 1000L, fixedDelay = 1000L) + public void execute_analysis_known() { + AuthInfo ai = new AuthInfo(); + ai.setTenant("monitor"); + + List records = new ArrayList<>(); + String name = "loganalysis"; + { + Record r = new Record(); + r.setName(name); + r.setTimestamp(System.currentTimeMillis()); + Map tags = new HashMap<>(); + tags.put("hostname", "hostname-1"); + tags.put("eventName", "test-1"); + r.setTags(tags); + + String value; + { + List logs = new ArrayList<>(); + + AnalyzedLog log1 = new AnalyzedLog(); + log1.setCount(2); + log1.setSample("2024-02-26 18:18:18 [INFO] hello world, cost=[100]"); + + logs.add(log1); + + value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs)); + } + + Map fields = new HashMap<>(); + fields.put("value", value); + r.setFields(fields); + records.add(r); + } + { + Record r = new Record(); + r.setName(name); + r.setTimestamp(System.currentTimeMillis()); + Map tags = new HashMap<>(); + tags.put("hostname", "hostname-2"); + tags.put("eventName", "test-1"); + r.setTags(tags); + + String value; + { + List logs = new ArrayList<>(); + + AnalyzedLog log1 = new AnalyzedLog(); + log1.setCount(3); + log1.setSample("2024-02-26 18:18:18 [INFO] hello world, cost=[100]"); + + logs.add(log1); + + value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs)); + } + + Map fields = new HashMap<>(); + fields.put("value", value); + r.setFields(fields); + records.add(r); + } + { + Record r = new Record(); + r.setName(name); + r.setTimestamp(System.currentTimeMillis()); + Map tags = new HashMap<>(); + tags.put("app", "testapp"); + tags.put("hostname", "hostname-3"); + tags.put("eventName", "__analysis"); + r.setTags(tags); + + String value; + { + List logs = new ArrayList<>(); + + AnalyzedLog log1 = new AnalyzedLog(); + List parts = new ArrayList<>(); + parts.add(new LAPart("UNKNOWN", false, false, 1)); + parts.add(new LAPart("hello world", false, true, 1)); + parts.add(new LAPart("cost", false, false, 1)); + parts.add(new LAPart("100", false, false, 1)); + log1.setParts(parts); + log1.setCount(3); + List sourceWords = new ArrayList<>(); + sourceWords.add(new SourceWord("1.1.1.1", 1)); + sourceWords.add(new SourceWord("2.2.2.2", 1)); + log1.setSourceWords(sourceWords); + log1.setSample("2024-02-26 18:18:18 [UNKNOWN] hello world, cost=[100]"); + + logs.add(log1); + + value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs)); + } + + Map fields = new HashMap<>(); + fields.put("value", value); + r.setFields(fields); + records.add(r); + } + + aggDispatcher.dispatchRecords(ai, name, records); + } +} diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AnalyzedLog.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AnalyzedLog.java new file mode 100644 index 000000000..89c88a68e --- /dev/null +++ b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AnalyzedLog.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.dispatcher.mock; + +import java.util.List; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class AnalyzedLog { + private List parts; + private String sample; + private int count; + private List sourceWords; +} diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/LAPart.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/LAPart.java new file mode 100644 index 000000000..1e4686a66 --- /dev/null +++ b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/LAPart.java @@ -0,0 +1,29 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.dispatcher.mock; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class LAPart { + private String content; + private boolean source; + private boolean important; + private int count; + + public LAPart() {} + + public LAPart(String content, boolean source, boolean important, int count) { + this.content = content; + this.source = source; + this.important = important; + this.count = count; + } +} diff --git a/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/SourceWord.java b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/SourceWord.java new file mode 100644 index 000000000..1a66ecf00 --- /dev/null +++ b/server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/SourceWord.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.dispatcher.mock; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +class SourceWord { + private String source; + private int count; +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AggTaskExecutor.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AggTaskExecutor.java index 4d8e49ad8..31026b9e8 100644 --- a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AggTaskExecutor.java +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AggTaskExecutor.java @@ -98,6 +98,12 @@ public AggTaskExecutor(AggTaskState state, CompletenessService completenessServi * @param aggTaskValue */ public void process(XAggTask latestAggTask, AggProtos.AggTaskValue aggTaskValue) { + boolean debug = latestAggTask.getInner().getExtension().isDebug(); + if (debug) { + log.info("[agg] [debug] [{}] input={}", key(), ProtoJsonUtils.toJson(aggTaskValue)); + } + + switch (aggTaskValue.getType()) { case AggTaskValueTypes.COMPLETENESS_INFO: processCompletenessInfo(latestAggTask, aggTaskValue); diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AnalyzedLog.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AnalyzedLog.java new file mode 100644 index 000000000..8e75d893e --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AnalyzedLog.java @@ -0,0 +1,143 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Data; + +/** + * After an analyzed log, similar logs will be aggregated together. + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class AnalyzedLog { + public static double SIMILAR_STRING_FACTOR = 0.9; + public static double SIMILAR_PARTS_FACTOR = 0.8; + public static int SOURCE_MAX_COUNT = 60; + public static int IGNORE_TYPE_LENGTH = 5; + + private List parts; + + /** + * One line of log sampling + */ + private String sample; + + /** + * Number of similar log lines + */ + private int count; + + private List sourceWords; + private Map ipCountMap = new HashMap<>(); + + @JsonIgnore + private transient Map sources; + + public void addIpCount(String hostname, int count) { + if (StringUtils.isEmpty(hostname)) { + return; + } + ipCountMap.put(hostname, ipCountMap.getOrDefault(hostname, 0) + count); + } + + public boolean isSimilarTo(StringBuilder reuse, AnalyzedLog al) { + return isSimilar(reuse, parts, al.parts); + } + + public void merge(AnalyzedLog al) { + // merge count + this.count += al.count; + + // merge ipCountMap + al.ipCountMap.forEach(this::addIpCount); + + // merge source words + if (sources == null) { + sources = new HashMap<>(); + for (SourceWord sw : sourceWords) { + sources.put(sw.getSource(), sw.getCount()); + } + } + for (SourceWord sw : al.getSourceWords()) { + sources.put(sw.getSource(), sources.getOrDefault(sw.getSource(), 0) + sw.getCount()); + } + } + + public void update() { + if (sources != null) { + List sourceWords = new ArrayList<>(); + sources.forEach((source, count) -> { + SourceWord sw = new SourceWord(); + sw.setSource(source); + sw.setCount(count); + sourceWords.add(sw); + }); + this.sourceWords = sourceWords; + } + } + + private static boolean isSimilar(StringBuilder reuse, List fs, List ts) { + double small, big; + + if (fs.size() > ts.size()) { + big = fs.size(); + small = ts.size(); + } else { + big = ts.size(); + small = fs.size(); + } + + if (small / big < SIMILAR_PARTS_FACTOR) { + return false; + } + + double similar = 0; + double total = 0; + for (LAPart f : fs) { + if (f.isSource()) { + continue; + } + if (f.getContent().length() < IGNORE_TYPE_LENGTH) { + continue; + } + boolean found = false; + for (LAPart t : ts) { + if (f.isSimilarTo(reuse, t)) { + found = true; + break; + } + } + int partSize = f.getContent().length(); + int s_add = 0, t_add = partSize; + if (found) { + s_add = partSize; + } + if (f.isImportant()) { + s_add *= 2; + } else { + t_add /= 2; + } + similar += s_add; + total += t_add; + } + if (total == 0) { + return true; + } + + return (similar / total) >= SIMILAR_PARTS_FACTOR; + } + +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/GroupField.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/GroupField.java index 940cf3a03..a42df717c 100644 --- a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/GroupField.java +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/GroupField.java @@ -38,6 +38,7 @@ public class GroupField { private TopnState topn; private HllState hll; private PercentileState percentile; + private LogAnalysisState logAnalysis; public GroupField() {} @@ -88,6 +89,15 @@ public void add(DataAccessor da, SelectItem si) { logSamples.add(ls.getSamples()); break; } + case AggFunc.TYPE_LOGANALYSIS_MERGE: { + if (logAnalysis == null) { + boolean unknownMode = "__analysis".equals(da.getTagOrDefault("eventName", "")); + logAnalysis = new LogAnalysisState(unknownMode); + } + LogAnalysis la = JsonUtils.fromJson(da.getStringField(), LogAnalysis.class); + logAnalysis.add(da, la); + break; + } case AggFunc.TYPE_TOPN: { if (topn == null) { AggFunc.TopnParams params = agg.getTopn(); @@ -166,6 +176,11 @@ public Object getFinalValue() { pfv.add(DEFAULT_PERCENTILE_RANKS[i], quantiles[i]); } return JsonUtils.toJson(pfv); + case AggFunc.TYPE_LOGANALYSIS_MERGE: + if (logAnalysis == null) { + return null; + } + return logAnalysis.getFinalValue(); default: throw new IllegalStateException("unsupported"); } diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/KnownValue.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/KnownValue.java new file mode 100644 index 000000000..ec302178f --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/KnownValue.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class KnownValue { + /** + * log sample + */ + private String sample; + /** + * log count + */ + private int count; + + private Map ipCountMap = new HashMap<>(); + + public void merge(String hostname, AnalyzedLog al) { + count += al.getCount(); + if (StringUtils.isNotEmpty(hostname)) { + ipCountMap.put(hostname, ipCountMap.getOrDefault(hostname, 0) + al.getCount()); + } + } +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LAPart.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LAPart.java new file mode 100644 index 000000000..3011ea74e --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LAPart.java @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +/** + * The part obtained after log analysis can be considered as a keyword in the log. + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class LAPart { + private String content; + private boolean source; + private boolean important; + private int count; + + /** + * The content of latterContent is content with all non-letter characters removed. + */ + @Getter(AccessLevel.NONE) + @Setter(AccessLevel.NONE) + private transient String letterContent; + + public String getLetterContent(StringBuilder reuse) { + if (letterContent != null) { + return letterContent; + } + + StringBuilder sb = reuse; + if (sb == null) { + sb = new StringBuilder(); + } + sb.setLength(0); + + for (int k = 0; k < content.length(); k++) { + char fc = content.charAt(k); + if (Character.isLetter(fc)) { + sb.append(fc); + } + } + + letterContent = sb.toString(); + return letterContent; + } + + public boolean isSimilarTo(StringBuilder reuse, LAPart t) { + String a = this.getLetterContent(reuse); + String b = t.getLetterContent(reuse); + return a.equals(b); + } +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysis.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysis.java new file mode 100644 index 000000000..637d7d39b --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysis.java @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import java.util.List; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class LogAnalysis { + private List analyzedLogs; +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysisState.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysisState.java new file mode 100644 index 000000000..c359cdd16 --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/LogAnalysisState.java @@ -0,0 +1,84 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import java.util.Collections; + +import org.apache.commons.collections4.CollectionUtils; + +import io.holoinsight.server.agg.v1.core.data.DataAccessor; +import lombok.NoArgsConstructor; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@NoArgsConstructor +public class LogAnalysisState { + private static final String HOSTNAME_TAG = "hostname"; + + private boolean unknownMode; + + private UnknownValue unknown; + private KnownValue known; + + public LogAnalysisState(boolean unknownMode) { + this.unknownMode = unknownMode; + } + + public void add(DataAccessor da, LogAnalysis la) { + if (la == null || CollectionUtils.isEmpty(la.getAnalyzedLogs())) { + return; + } + if (unknownMode) { + addUnknown(da, la); + } else { + addKnown(da, la); + } + } + + private void addUnknown(DataAccessor da, LogAnalysis la) { + if (unknown == null) { + unknown = new UnknownValue(); + } + + String hostname = da.getTag(HOSTNAME_TAG); + + int totalCount = 0; + StringBuilder reuse = new StringBuilder(); + for (AnalyzedLog al : la.getAnalyzedLogs()) { + totalCount += al.getCount(); + al.addIpCount(hostname, al.getCount()); + unknown.merge(reuse, al); + } + + unknown.addIpCount(hostname, totalCount); + } + + private void addKnown(DataAccessor da, LogAnalysis la) { + // 'known' has only one analyzed log + AnalyzedLog al = la.getAnalyzedLogs().get(0); + + if (known == null) { + known = new KnownValue(); + known.setSample(al.getSample()); + } + + String hostname = da.getTag(HOSTNAME_TAG); + known.merge(hostname, al); + } + + public Object getFinalValue() { + if (unknown != null) { + unknown.update(); + return Collections.singletonMap("unknown", unknown); + } + if (known != null) { + return Collections.singletonMap("known", known); + } + return null; + } +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/SourceWord.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/SourceWord.java new file mode 100644 index 000000000..6fb3aaf7d --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/SourceWord.java @@ -0,0 +1,18 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class SourceWord { + private String source; + private int count; +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/UnknownValue.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/UnknownValue.java new file mode 100644 index 000000000..9cd30a217 --- /dev/null +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/UnknownValue.java @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.executor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import lombok.Data; + +/** + *

+ * created at 2024/2/26 + * + * @author xzchaoo + */ +@Data +class UnknownValue { + private MergeData mergeData = new MergeData(); + private Map ipCountMap = new HashMap<>(); + + public void addIpCount(String hostname, int count) { + if (StringUtils.isEmpty(hostname)) { + return; + } + ipCountMap.put(hostname, ipCountMap.getOrDefault(hostname, 0) + count); + } + + public void merge(StringBuilder reuse, AnalyzedLog al) { + for (AnalyzedLog exist : mergeData.analyzedLogs) { + if (exist.isSimilarTo(reuse, al)) { + exist.merge(al); + return; + } + } + mergeData.analyzedLogs.add(al); + } + + public void update() { + mergeData.analyzedLogs.forEach(AnalyzedLog::update); + } + + static class MergeData { + List analyzedLogs = new ArrayList<>(); + } +} diff --git a/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/AggCenter.java b/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/AggCenter.java index 6e739130c..22ed3d0b7 100644 --- a/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/AggCenter.java +++ b/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/AggCenter.java @@ -3,20 +3,22 @@ */ package io.holoinsight.server.query.service.analysis; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import org.apache.commons.collections.CollectionUtils; + import com.google.gson.reflect.TypeToken; + +import io.holoinsight.server.apm.common.utils.GsonUtils; import io.holoinsight.server.query.service.analysis.collect.AnalyzedLog; import io.holoinsight.server.query.service.analysis.collect.MergeData; import io.holoinsight.server.query.service.analysis.known.KnownValue; import io.holoinsight.server.query.service.analysis.unknown.UnknownValue; -import io.holoinsight.server.apm.common.utils.GsonUtils; import io.holoinsight.server.query.service.sample.LogSample; import io.holoinsight.server.query.service.sample.LogSamples; -import org.apache.commons.collections.CollectionUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.BiFunction; public class AggCenter { private static final Map, String, Mergeable>> AGGREGATORS = @@ -24,8 +26,13 @@ public class AggCenter { static { AGGREGATORS.put("unknown-analysis", (tags, json) -> { - UnknownValue value = new UnknownValue(); Analysis analysis = GsonUtils.fromJson(json, Analysis.class); + + // If the pre-aggregated result is not empty, it is returned directly. + if (analysis != null && analysis.getUnknown() != null) { + return analysis.getUnknown(); + } + UnknownValue value = new UnknownValue(); if (analysis != null && CollectionUtils.isNotEmpty(analysis.getAnalyzedLogs())) { List logs = analysis.getAnalyzedLogs(); String host = tags.getOrDefault("hostname", "UNKNOWN"); @@ -45,8 +52,12 @@ public class AggCenter { }); AGGREGATORS.put("known-analysis", (tags, json) -> { - KnownValue value = null; Analysis analysis = GsonUtils.fromJson(json, Analysis.class); + // If the pre-aggregated result is not empty, it is returned directly. + if (analysis != null && analysis.getKnown() != null) { + return analysis.getKnown(); + } + KnownValue value = null; if (analysis != null && CollectionUtils.isNotEmpty(analysis.getAnalyzedLogs())) { List logs = analysis.getAnalyzedLogs(); AnalyzedLog log = logs.get(0); diff --git a/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/Analysis.java b/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/Analysis.java index 6ef111a9f..f42c23144 100644 --- a/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/Analysis.java +++ b/server/query/query-service/src/main/java/io/holoinsight/server/query/service/analysis/Analysis.java @@ -3,13 +3,15 @@ */ package io.holoinsight.server.query.service.analysis; +import java.util.List; + import io.holoinsight.server.query.service.analysis.collect.AnalyzedLog; +import io.holoinsight.server.query.service.analysis.known.KnownValue; +import io.holoinsight.server.query.service.analysis.unknown.UnknownValue; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; - /** * @author xiangwanpeng * @version : Analysis.java, v 0.1 2022年12月08日 17:22 xiangwanpeng Exp $ @@ -20,4 +22,14 @@ public class Analysis { private int count; private List analyzedLogs; + + /** + * The result after pre-aggregation + */ + private KnownValue known; + + /** + * The result after pre-aggregation + */ + private UnknownValue unknown; }