From 84ff87826a139e7b07565e8f753791b2ccc5968f Mon Sep 17 00:00:00 2001 From: zl11357 Date: Mon, 29 Jan 2024 14:42:16 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DgenerateTimeFieldSqlW?= =?UTF-8?q?ithFormatDateTime64ZoneShangHai=20=20toDateTime64=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E8=BD=AC=E6=8D=A2=E7=BC=BA=E5=A4=B1=E7=B2=BE=E5=BA=A6?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.fix:修复generateTimeFieldSqlWithFormatDateTime64ZoneShangHai toDateTime64方法转换缺失精度问题 2.refactor:将isDateTime64Ms和isDateTime方法提取到SqlUtils独立方法 3.refactor:重命名generateTimeFieldSqlWithFormatUnixTimestamp64Milli方法，改为generateTimeFieldSqlWithFormatUnixTimestamp64 --- .../ly/ckibana/parser/HitsResultParser.java | 2 +- .../aggs/DateHistogramAggregation.java | 2 +- .../java/com/ly/ckibana/util/ProxyUtils.java | 32 ++++------------- .../java/com/ly/ckibana/util/SqlUtils.java | 35 +++++++++++++++++++ .../ly/ckibana/converter/CommonAggTest.java | 16 ++++----- .../ly/ckibana/converter/CommonDSLTest.java | 16 ++++----- .../converter/CommonTermsAggOrderypeTest.java | 14 ++++---- .../converter/QueryStringClauseTest.java | 16 ++++----- 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/ly/ckibana/parser/HitsResultParser.java b/src/main/java/com/ly/ckibana/parser/HitsResultParser.java index 49ff466..d2a2d1e 100644 --- a/src/main/java/com/ly/ckibana/parser/HitsResultParser.java +++ b/src/main/java/com/ly/ckibana/parser/HitsResultParser.java @@ -291,7 +291,7 @@ private HitsOptimizedResult doOptimizeSearchTimeRange(CkRequestContext ckRequest */ private CkRequest getByMinuteRequest(SortType sortType, CkRequestContext ckRequestContext, Range orgTimeClause) { CkRequest byMinuteRequest = ProxyUtils.buildRequest(ckRequestContext.getTableName(), SqlConstants.COUNT_QUERY); - String timeField = ProxyUtils.generateTimeFieldSqlWithFormatUnixTimestamp64Milli(ckRequestContext.getIndexPattern().getTimeField(), orgTimeClause.getCkFieldType()); + String timeField = ProxyUtils.generateTimeFieldSqlWithFormatUnixTimestamp64(ckRequestContext.getIndexPattern().getTimeField(), orgTimeClause.getCkFieldType()); byMinuteRequest.appendSelect(String.format(SqlConstants.TIME_AGG_BY_MINUTE_TEMPLATE, timeField, SqlConstants.CK_MINUTE_NAME)); byMinuteRequest.initGroupBy(SqlConstants.CK_MINUTE_NAME); byMinuteRequest.orderBy(SqlConstants.CK_MINUTE_NAME + " " + sortType.name()); diff --git a/src/main/java/com/ly/ckibana/strategy/aggs/DateHistogramAggregation.java b/src/main/java/com/ly/ckibana/strategy/aggs/DateHistogramAggregation.java index ace236c..230414c 100644 --- a/src/main/java/com/ly/ckibana/strategy/aggs/DateHistogramAggregation.java +++ b/src/main/java/com/ly/ckibana/strategy/aggs/DateHistogramAggregation.java @@ -67,7 +67,7 @@ public List buildSelectSqlConvertors(Range timeRange) { FieldSqlConverter fieldAggregation = new FieldSqlConverter(); fieldAggregation.setCondition( SqlUtils.getFunctionString(SqlConstants.TO_INT64, "(%s) / %d", - ProxyUtils.generateTimeFieldSqlWithFormatUnixTimestamp64Milli(getField(), timeRange.getCkFieldType()), + ProxyUtils.generateTimeFieldSqlWithFormatUnixTimestamp64(getField(), timeRange.getCkFieldType()), getInterval()) ); fieldAggregation.setName(queryFieldName()); diff --git a/src/main/java/com/ly/ckibana/util/ProxyUtils.java b/src/main/java/com/ly/ckibana/util/ProxyUtils.java index b2606ea..0d285e6 100644 --- a/src/main/java/com/ly/ckibana/util/ProxyUtils.java +++ b/src/main/java/com/ly/ckibana/util/ProxyUtils.java @@ -234,10 +234,10 @@ public static String getErrorResponse(String message) { /** * 时间查询sql转换，支持字符串（耗性能，不推荐）和数值类型,DateTime, DateTime64类型. */ - public static String generateTimeFieldSqlWithFormatUnixTimestamp64Milli(String ckFieldName, String ckFieldType) { - if (isDateTime64Ms(ckFieldType)) { + public static String generateTimeFieldSqlWithFormatUnixTimestamp64(String ckFieldName, String ckFieldType) { + if (SqlUtils.isDateTime64(ckFieldType)) { return String.format("toUnixTimestamp64Milli(%s)", getFieldSqlPart(ckFieldName)); - } else if (isDateTime(ckFieldType)) { + } else if (SqlUtils.isDateTime(ckFieldType)) { return String.format("toUnixTimestamp(%s)*1000", getFieldSqlPart(ckFieldName)); } else { return getFieldSqlPart(ckFieldName); @@ -274,33 +274,13 @@ public static Range getRangeWrappedBySqlFunction(Range orgRange, boolean isTimeF return rangeConverted; } - /** - * 是否为DateTime64时间类型. - * - * @param ckFieldType ck字段类型 - * @return true:是DateTime64类型 - */ - public static boolean isDateTime64Ms(String ckFieldType) { - return StringUtils.startsWith(ckFieldType, SqlConstants.TYPE_DATETIME64); - } - - /** - * 是否为DateTime时间类型. - * - * @param ckFieldType ck字段类型 - * @return true:是DateTime类型 - */ - public static boolean isDateTime(String ckFieldType) { - return StringUtils.startsWith(ckFieldType, SqlConstants.TYPE_DATETIME); - } - /** * 时间字段转换。作为值 or 字段. */ public static String generateTimeFieldSqlWithFormatDateTime64ZoneShangHai(Object value, String ckFieldType) { - if (isDateTime64Ms(ckFieldType)) { - return String.format("toDateTime64(%s/1000.0)", value.toString()); - } else if (isDateTime(ckFieldType)) { + if (SqlUtils.isDateTime64(ckFieldType)) { + return String.format("toDateTime64(%s/1000,%d)", value.toString(),SqlUtils.getDateTime64Scale(ckFieldType)); + } else if (SqlUtils.isDateTime(ckFieldType)) { return String.format("toDateTime(%s/1000)", value.toString()); } else { return value.toString(); diff --git a/src/main/java/com/ly/ckibana/util/SqlUtils.java b/src/main/java/com/ly/ckibana/util/SqlUtils.java index 8d36f5b..1949923 100644 --- a/src/main/java/com/ly/ckibana/util/SqlUtils.java +++ b/src/main/java/com/ly/ckibana/util/SqlUtils.java @@ -406,4 +406,39 @@ public static String generateIpSql(Object value, IPType ipType, boolean isQueryV public static String getSelectTemplate(String selectSql, String table) { return String.format("SELECT %s FROM %s", selectSql, table); } + + + /** + * 是否为DateTime64时间类型. + * + * @param ckFieldType ck字段类型 + * @return true:是DateTime64类型 + */ + public static boolean isDateTime64(String ckFieldType) { + return StringUtils.startsWith(ckFieldType, SqlConstants.TYPE_DATETIME64); + } + + + /** + * DateTime64时间类型，提取精度 + * + * @param ckFieldType ck字段类型,如DateTime64(3) + * @return 精度,如3 + */ + public static int getDateTime64Scale(String ckFieldType) { + String scale= ckFieldType.replace(SqlConstants.TYPE_DATETIME64,StringUtils.EMPTY) + .replace(Constants.Symbol.LEFT_PARENTHESIS,StringUtils.EMPTY) + .replace(Constants.Symbol.RIGHT_PARENTHESIS,StringUtils.EMPTY); + return Integer.parseInt(scale); + } + /** + * 是否为DateTime时间类型. + * + * @param ckFieldType ck字段类型 + * @return true:是DateTime类型 + */ + public static boolean isDateTime(String ckFieldType) { + return StringUtils.startsWith(ckFieldType, SqlConstants.TYPE_DATETIME); + } + } diff --git a/src/test/java/com/ly/ckibana/converter/CommonAggTest.java b/src/test/java/com/ly/ckibana/converter/CommonAggTest.java index b17a665..4fa2abf 100644 --- a/src/test/java/com/ly/ckibana/converter/CommonAggTest.java +++ b/src/test/java/com/ly/ckibana/converter/CommonAggTest.java @@ -43,8 +43,8 @@ public class CommonAggTest extends CommonTest { public void testFiltersAndDateHistogramAgg() { String query = "{\"aggs\":{\"2\":{\"filters\":{\"filters\":{\"s1:\\\"s1value1\\\"\":{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},\"s2:\\\"s1value2\\\"\":{\"query_string\":{\"query\":\"s2:\\\"s1value2\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}}}},\"aggs\":{\"3\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890656331,\"lte\":1697891556331,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `3_@timestampDateTime`,count(1) as `3__ckCount`,count(1) as `s1:\\\"s1value1\\\"__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891550000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890650000/1000.0) ) AND ( ( `s1` like '%s1value1%' ) ) GROUP BY `3_@timestampDateTime` ORDER BY `3_@timestampDateTime` asc LIMIT 30000\",\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `3_@timestampDateTime`,count(1) as `3__ckCount`,count(1) as `s2:\\\"s1value2\\\"__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891550000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890650000/1000.0) ) AND ( ( `s2` like '%s1value2%' ) ) GROUP BY `3_@timestampDateTime` ORDER BY `3_@timestampDateTime` asc LIMIT 30000\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `3_@timestampDateTime`,count(1) as `3__ckCount`,count(1) as `s1:\\\"s1value1\\\"__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891550000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890650000/1000,3) ) AND ( ( `s1` like '%s1value1%' ) ) GROUP BY `3_@timestampDateTime` ORDER BY `3_@timestampDateTime` asc LIMIT 30000\",\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `3_@timestampDateTime`,count(1) as `3__ckCount`,count(1) as `s2:\\\"s1value2\\\"__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891550000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890650000/1000,3) ) AND ( ( `s2` like '%s1value2%' ) ) GROUP BY `3_@timestampDateTime` ORDER BY `3_@timestampDateTime` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_AGGS_FILTERS_AND_DATE_HISTOGRAM_AGG, query, Boolean.FALSE, expectedSqls); } @@ -56,7 +56,7 @@ public void testFiltersAndDateHistogramAgg() { public void testDateHistogramAgg() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1:\\\"s1value\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1:\\\"s1value\\\"\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890734041,\"lte\":1697891634041,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( ( `s1` like '%s1value%' ) AND ( `s1` like '%s1value%' ) ) AND (( `@timestampDateTime` <= toDateTime64(1697891630000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890730000/1000.0) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( ( `s1` like '%s1value%' ) AND ( `s1` like '%s1value%' ) ) AND (( `@timestampDateTime` <= toDateTime64(1697891630000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890730000/1000,3) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_AGGS_DATE_HISTOGRAM_AGG, query, Boolean.FALSE, expectedSqls); @@ -69,7 +69,7 @@ public void testDateHistogramAgg() { public void testTermsAgg() { String query = "{\"aggs\":{\"2\":{\"terms\":{\"field\":\"s1\",\"size\":10,\"order\":{\"_count\":\"desc\"}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697891206239,\"lte\":1697892106239,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT `s1` as `2_s1`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892100000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697891200000/1000.0) ) AND (`2_s1` is not null AND `2_s1` != '' ) GROUP BY `2_s1` ORDER BY `2__ckCount` desc LIMIT 10\"\n" + + " \"SELECT `s1` as `2_s1`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892100000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697891200000/1000,3) ) AND (`2_s1` is not null AND `2_s1` != '' ) GROUP BY `2_s1` ORDER BY `2__ckCount` desc LIMIT 10\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG, query, Boolean.FALSE, expectedSqls); } @@ -88,7 +88,7 @@ public void testTermsAgg() { public void testMathAndPercentAndDateHistogramAgg() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"sum\":{\"field\":\"i1\"}},\"4\":{\"min\":{\"field\":\"i2\"}},\"5\":{\"max\":{\"field\":\"i3\"}},\"6\":{\"avg\":{\"field\":\"i1\"}},\"7\":{\"percentiles\":{\"field\":\"i1\",\"percents\":[90,95,99],\"keyed\":false}},\"8\":{\"percentile_ranks\":{\"field\":\"i2\",\"values\":[100],\"keyed\":false}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697891258636,\"lte\":1697892158636,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT SUM(`i1`) as `3_i1`,MIN(`i2`) as `4_i2`,MAX(`i3`) as `5_i3`,avg(`i1`) as `6_i1`,quantile(0.9)(`i1`) as `i1_90`,quantile(0.95)(`i1`) as `i1_95`,quantile(0.99)(`i1`) as `i1_99`,count(1) as `7__ckCount`,countIf(`i2` <= 100) as `i2_100`,count(1) as `8__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892150000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697891250000/1000.0) ) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + + " \"SELECT SUM(`i1`) as `3_i1`,MIN(`i2`) as `4_i2`,MAX(`i3`) as `5_i3`,avg(`i1`) as `6_i1`,quantile(0.9)(`i1`) as `i1_90`,quantile(0.95)(`i1`) as `i1_95`,quantile(0.99)(`i1`) as `i1_99`,count(1) as `7__ckCount`,countIf(`i2` <= 100) as `i2_100`,count(1) as `8__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892150000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697891250000/1000,3) ) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_MATH_AND_PERCENTILE_DATE_HISTOGRAM_AGG, query, Boolean.FALSE, expectedSqls); } @@ -106,7 +106,7 @@ public void testMathAndPercentAndDateHistogramAgg() { public void testMathAndPercentAgg() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"sum\":{\"field\":\"i1\"}},\"4\":{\"min\":{\"field\":\"i2\"}},\"5\":{\"max\":{\"field\":\"i3\"}},\"6\":{\"avg\":{\"field\":\"i1\"}},\"7\":{\"percentiles\":{\"field\":\"i1\",\"percents\":[90,95,99],\"keyed\":false}},\"8\":{\"percentile_ranks\":{\"field\":\"i2\",\"values\":[100],\"keyed\":false}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697891258636,\"lte\":1697892158636,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT SUM(`i1`) as `3_i1`,MIN(`i2`) as `4_i2`,MAX(`i3`) as `5_i3`,avg(`i1`) as `6_i1`,quantile(0.9)(`i1`) as `i1_90`,quantile(0.95)(`i1`) as `i1_95`,quantile(0.99)(`i1`) as `i1_99`,count(1) as `7__ckCount`,countIf(`i2` <= 100) as `i2_100`,count(1) as `8__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892150000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697891250000/1000.0) ) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + + " \"SELECT SUM(`i1`) as `3_i1`,MIN(`i2`) as `4_i2`,MAX(`i3`) as `5_i3`,avg(`i1`) as `6_i1`,quantile(0.9)(`i1`) as `i1_90`,quantile(0.95)(`i1`) as `i1_95`,quantile(0.99)(`i1`) as `i1_99`,count(1) as `7__ckCount`,countIf(`i2` <= 100) as `i2_100`,count(1) as `8__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892150000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697891250000/1000,3) ) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_MATH_AND_PERCENTILE_AGG, query, Boolean.FALSE, expectedSqls); } @@ -119,7 +119,7 @@ public void testMathAndPercentAgg() { public void testRangeNumber() { String query = "{\"aggs\":{\"2\":{\"range\":{\"field\":\"i1\",\"ranges\":[{\"from\":0,\"to\":1000},{\"from\":1000,\"to\":2000},{\"from\":3000}],\"keyed\":true}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697891642926,\"lte\":1697892542926,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = " [\n" + - " \"SELECT countIf(`i1` >= 0 AND `i1` < 1000) as `0-1000`,countIf(`i1` >= 1000 AND `i1` < 2000) as `1000-2000`,countIf(`i1` >= 3000) as `3000-*` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892540000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697891640000/1000.0) ) AND (`i1` != 2147483647) LIMIT 30000\"\n" + + " \"SELECT countIf(`i1` >= 0 AND `i1` < 1000) as `0-1000`,countIf(`i1` >= 1000 AND `i1` < 2000) as `1000-2000`,countIf(`i1` >= 3000) as `3000-*` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892540000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697891640000/1000,3) ) AND (`i1` != 2147483647) LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_RANGE_NUMBER_AGG, query, Boolean.FALSE, expectedSqls); } @@ -131,7 +131,7 @@ public void testRangeNumber() { public void testCardinality() { String query = "{\"aggs\":{\"2\":{\"range\":{\"field\":\"i1\",\"ranges\":[{\"from\":0,\"to\":100},{\"from\":100,\"to\":200},{\"from\":200,\"to\":300}],\"keyed\":true}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697891714119,\"lte\":1697892614119,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT countIf(`i1` >= 0 AND `i1` < 100) as `0-100`,countIf(`i1` >= 100 AND `i1` < 200) as `100-200`,countIf(`i1` >= 200 AND `i1` < 300) as `200-300` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892610000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697891710000/1000.0) ) AND (`i1` != 2147483647) LIMIT 30000\"\n" + + " \"SELECT countIf(`i1` >= 0 AND `i1` < 100) as `0-100`,countIf(`i1` >= 100 AND `i1` < 200) as `100-200`,countIf(`i1` >= 200 AND `i1` < 300) as `200-300` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697892610000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697891710000/1000,3) ) AND (`i1` != 2147483647) LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_CARDINALITY_AGG, query, Boolean.FALSE, expectedSqls); } diff --git a/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java b/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java index 8f6577d..fae2638 100644 --- a/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java +++ b/src/test/java/com/ly/ckibana/converter/CommonDSLTest.java @@ -36,8 +36,8 @@ public class CommonDSLTest extends CommonTest { public void testIntegerDSL() { String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"i1\":{\"query\":1}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"i1\":\"3\"}},{\"match_phrase\":{\"i1\":\"4\"}}],\"minimum_should_match\":1}},{\"range\":{\"i1\":{\"gte\":7,\"lt\":8}}},{\"exists\":{\"field\":\"i1\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697892216652,\"lte\":1697893116652,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[{\"match_phrase\":{\"i1\":{\"query\":2}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"i1\":\"5\"}},{\"match_phrase\":{\"i1\":\"6\"}}],\"minimum_should_match\":1}},{\"range\":{\"i1\":{\"gte\":9,\"lt\":10}}},{\"exists\":{\"field\":\"i2\"}}]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`i1` = 1) AND ((`i1` = 3) OR (`i1` = 4)) AND ( `i1` < 8 AND `i1` >= 7 ) AND (isNotNull(`i1`))) AND ( NOT (`i1` = 2) AND NOT ((`i1` = 5) OR (`i1` = 6)) AND NOT ( `i1` < 10 AND `i1` >= 9 ) AND NOT (isNotNull(`i2`))) AND (( `@timestampDateTime` <= toDateTime64(1697893110000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697892210000/1000.0) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + - " \"SELECT * FROM `table1_all` PREWHERE ((`i1` = 1) AND ((`i1` = 3) OR (`i1` = 4)) AND ( `i1` < 8 AND `i1` >= 7 ) AND (isNotNull(`i1`))) AND ( NOT (`i1` = 2) AND NOT ((`i1` = 5) OR (`i1` = 6)) AND NOT ( `i1` < 10 AND `i1` >= 9 ) AND NOT (isNotNull(`i2`))) AND (( `@timestampDateTime` <= toDateTime64(1697893110000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697892210000/1000.0) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`i1` = 1) AND ((`i1` = 3) OR (`i1` = 4)) AND ( `i1` < 8 AND `i1` >= 7 ) AND (isNotNull(`i1`))) AND ( NOT (`i1` = 2) AND NOT ((`i1` = 5) OR (`i1` = 6)) AND NOT ( `i1` < 10 AND `i1` >= 9 ) AND NOT (isNotNull(`i2`))) AND (( `@timestampDateTime` <= toDateTime64(1697893110000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697892210000/1000,3) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + + " \"SELECT * FROM `table1_all` PREWHERE ((`i1` = 1) AND ((`i1` = 3) OR (`i1` = 4)) AND ( `i1` < 8 AND `i1` >= 7 ) AND (isNotNull(`i1`))) AND ( NOT (`i1` = 2) AND NOT ((`i1` = 5) OR (`i1` = 6)) AND NOT ( `i1` < 10 AND `i1` >= 9 ) AND NOT (isNotNull(`i2`))) AND (( `@timestampDateTime` <= toDateTime64(1697893110000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697892210000/1000,3) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + " ]"; doTest(TEST_INTEGER_DSL, query, Boolean.TRUE, expectedSqls); @@ -50,8 +50,8 @@ public void testIntegerDSL() { public void testStringDSL() { String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"s1\":{\"query\":\"sv1\"}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"s1\":\"sv3\"}},{\"match_phrase\":{\"s1\":\"sv4\"}}],\"minimum_should_match\":1}},{\"exists\":{\"field\":\"s1\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697892781326,\"lte\":1697893681326,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[{\"match_phrase\":{\"s1\":{\"query\":\"sv2\"}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"s1\":\"sv5\"}},{\"match_phrase\":{\"s1\":\"sv6\"}}],\"minimum_should_match\":1}},{\"exists\":{\"field\":\"s2\"}}]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"120000ms\"}\n"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`s1` like '%sv1%') AND ((`s1` like '%sv3%') OR (`s1` like '%sv4%')) AND (isNotNull(`s1`) AND `s1` != '')) AND ( NOT (`s1` like '%sv2%') AND NOT ((`s1` like '%sv5%') OR (`s1` like '%sv6%')) AND NOT (isNotNull(`s2`) AND `s2` != '')) AND (( `@timestampDateTime` <= toDateTime64(1697893680000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697892780000/1000.0) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + - " \"SELECT * FROM `table1_all` PREWHERE ((`s1` like '%sv1%') AND ((`s1` like '%sv3%') OR (`s1` like '%sv4%')) AND (isNotNull(`s1`) AND `s1` != '')) AND ( NOT (`s1` like '%sv2%') AND NOT ((`s1` like '%sv5%') OR (`s1` like '%sv6%')) AND NOT (isNotNull(`s2`) AND `s2` != '')) AND (( `@timestampDateTime` <= toDateTime64(1697893680000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697892780000/1000.0) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`s1` like '%sv1%') AND ((`s1` like '%sv3%') OR (`s1` like '%sv4%')) AND (isNotNull(`s1`) AND `s1` != '')) AND ( NOT (`s1` like '%sv2%') AND NOT ((`s1` like '%sv5%') OR (`s1` like '%sv6%')) AND NOT (isNotNull(`s2`) AND `s2` != '')) AND (( `@timestampDateTime` <= toDateTime64(1697893680000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697892780000/1000,3) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + + " \"SELECT * FROM `table1_all` PREWHERE ((`s1` like '%sv1%') AND ((`s1` like '%sv3%') OR (`s1` like '%sv4%')) AND (isNotNull(`s1`) AND `s1` != '')) AND ( NOT (`s1` like '%sv2%') AND NOT ((`s1` like '%sv5%') OR (`s1` like '%sv6%')) AND NOT (isNotNull(`s2`) AND `s2` != '')) AND (( `@timestampDateTime` <= toDateTime64(1697893680000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697892780000/1000,3) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + " ]"; doTest(TEST_STRING_DSL, query, Boolean.TRUE, expectedSqls); @@ -64,8 +64,8 @@ public void testStringDSL() { public void testTimeFieldDSL() { String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"@timestampDateTime\":{\"query\":\"2023-10-23 21:05:30.000\"}}},{\"bool\":{\"minimum_should_match\":1,\"should\":[{\"match_phrase\":{\"@timestampDateTime\":\"2023-10-23 20:02:00.000\"}},{\"match_phrase\":{\"@timestampDateTime\":\"2023-10-23 20:03:00.000\"}}]}},{\"range\":{\"@timestampDateTime\":{\"gte\":\"2023-10-23 20:06:00.000\",\"lt\":\"2023-10-23 20:07:00.000\"}}},{\"exists\":{\"field\":\"@timestampDateTime\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1698066374405,\"lte\":1698067274405,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[{\"match_phrase\":{\"@timestampDateTime\":{\"query\":\"2023-10-23 20:01:00.000\"}}},{\"bool\":{\"minimum_should_match\":1,\"should\":[{\"match_phrase\":{\"@timestampDateTime\":\"2023-10-23 20:04:00.000\"}},{\"match_phrase\":{\"@timestampDateTime\":\"2023-10-23 20:05:00.000\"}}]}},{\"range\":{\"@timestampDateTime\":{\"gte\":\"2023-10-23 20:08:00.000\",\"lt\":\"2023-10-23 20:09:00.000\"}}}]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"300000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`@timestampDateTime` = '2023-10-23 21:05:30.000') AND ((`@timestampDateTime` = '2023-10-23 20:02:00.000') OR (`@timestampDateTime` = '2023-10-23 20:03:00.000')) AND (isNotNull(`@timestampDateTime`))) AND ( NOT (`@timestampDateTime` = '2023-10-23 20:01:00.000') AND NOT ((`@timestampDateTime` = '2023-10-23 20:04:00.000') OR (`@timestampDateTime` = '2023-10-23 20:05:00.000'))) AND (( `@timestampDateTime` < toDateTime64(1698062940000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698062880000/1000.0) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + - " \"SELECT * FROM `table1_all` PREWHERE ((`@timestampDateTime` = '2023-10-23 21:05:30.000') AND ((`@timestampDateTime` = '2023-10-23 20:02:00.000') OR (`@timestampDateTime` = '2023-10-23 20:03:00.000')) AND (isNotNull(`@timestampDateTime`))) AND ( NOT (`@timestampDateTime` = '2023-10-23 20:01:00.000') AND NOT ((`@timestampDateTime` = '2023-10-23 20:04:00.000') OR (`@timestampDateTime` = '2023-10-23 20:05:00.000'))) AND (( `@timestampDateTime` < toDateTime64(1698062940000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698062880000/1000.0) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`@timestampDateTime` = '2023-10-23 21:05:30.000') AND ((`@timestampDateTime` = '2023-10-23 20:02:00.000') OR (`@timestampDateTime` = '2023-10-23 20:03:00.000')) AND (isNotNull(`@timestampDateTime`))) AND ( NOT (`@timestampDateTime` = '2023-10-23 20:01:00.000') AND NOT ((`@timestampDateTime` = '2023-10-23 20:04:00.000') OR (`@timestampDateTime` = '2023-10-23 20:05:00.000'))) AND (( `@timestampDateTime` < toDateTime64(1698062940000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698062880000/1000,3) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + + " \"SELECT * FROM `table1_all` PREWHERE ((`@timestampDateTime` = '2023-10-23 21:05:30.000') AND ((`@timestampDateTime` = '2023-10-23 20:02:00.000') OR (`@timestampDateTime` = '2023-10-23 20:03:00.000')) AND (isNotNull(`@timestampDateTime`))) AND ( NOT (`@timestampDateTime` = '2023-10-23 20:01:00.000') AND NOT ((`@timestampDateTime` = '2023-10-23 20:04:00.000') OR (`@timestampDateTime` = '2023-10-23 20:05:00.000'))) AND (( `@timestampDateTime` < toDateTime64(1698062940000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698062880000/1000,3) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + " ]"; doTest(TEST_DATETIME64_DSL, query, Boolean.TRUE, expectedSqls); } @@ -77,8 +77,8 @@ public void testTimeFieldDSL() { public void testIpv4OrIpv6DSL() { String query = "{\"version\":true,\"size\":500,\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"_source\":{\"excludes\":[]},\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1}}},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_phrase\":{\"ipv4\":{\"query\":\"1.1.1.1\"}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"ipv4\":\"3.3.3.3\"}},{\"match_phrase\":{\"ipv4\":\"4.4.4.4\"}}],\"minimum_should_match\":1}},{\"range\":{\"ipv4\":{\"gte\":\"7.7.7.7\",\"lt\":\"8.8.8.8\"}}},{\"exists\":{\"field\":\"ipv6\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1698056661828,\"lte\":1698057561828,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[{\"match_phrase\":{\"ipv4\":{\"query\":\"2.2.2.2\"}}},{\"bool\":{\"should\":[{\"match_phrase\":{\"ipv4\":\"5.5.5.5\"}},{\"match_phrase\":{\"ipv4\":\"6.6.6.6\"}}],\"minimum_should_match\":1}},{\"range\":{\"ipv4\":{\"gte\":\"9.9.9.9\",\"lt\":\"10.10.10.10\"}}},{\"exists\":{\"field\":\"ipv4\"}}]}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND ((`ipv4` = IPv4StringToNumOrDefault('3.3.3.3')) OR (`ipv4` = IPv4StringToNumOrDefault('4.4.4.4'))) AND ( `ipv4` < IPv4StringToNumOrDefault('8.8.8.8') AND `ipv4` >= IPv4StringToNumOrDefault('7.7.7.7') ) AND (isNotNull(`ipv6`))) AND ( NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND NOT ((`ipv4` = IPv4StringToNumOrDefault('5.5.5.5')) OR (`ipv4` = IPv4StringToNumOrDefault('6.6.6.6'))) AND NOT ( `ipv4` < IPv4StringToNumOrDefault('10.10.10.10') AND `ipv4` >= IPv4StringToNumOrDefault('9.9.9.9') ) AND NOT (isNotNull(`ipv4`))) AND (( `@timestampDateTime` <= toDateTime64(1698057560000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698056660000/1000.0) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + - " \"SELECT * FROM `table1_all` PREWHERE ((`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND ((`ipv4` = IPv4StringToNumOrDefault('3.3.3.3')) OR (`ipv4` = IPv4StringToNumOrDefault('4.4.4.4'))) AND ( `ipv4` < IPv4StringToNumOrDefault('8.8.8.8') AND `ipv4` >= IPv4StringToNumOrDefault('7.7.7.7') ) AND (isNotNull(`ipv6`))) AND ( NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND NOT ((`ipv4` = IPv4StringToNumOrDefault('5.5.5.5')) OR (`ipv4` = IPv4StringToNumOrDefault('6.6.6.6'))) AND NOT ( `ipv4` < IPv4StringToNumOrDefault('10.10.10.10') AND `ipv4` >= IPv4StringToNumOrDefault('9.9.9.9') ) AND NOT (isNotNull(`ipv4`))) AND (( `@timestampDateTime` <= toDateTime64(1698057560000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698056660000/1000.0) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + + " \"SELECT toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ((`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND ((`ipv4` = IPv4StringToNumOrDefault('3.3.3.3')) OR (`ipv4` = IPv4StringToNumOrDefault('4.4.4.4'))) AND ( `ipv4` < IPv4StringToNumOrDefault('8.8.8.8') AND `ipv4` >= IPv4StringToNumOrDefault('7.7.7.7') ) AND (isNotNull(`ipv6`))) AND ( NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND NOT ((`ipv4` = IPv4StringToNumOrDefault('5.5.5.5')) OR (`ipv4` = IPv4StringToNumOrDefault('6.6.6.6'))) AND NOT ( `ipv4` < IPv4StringToNumOrDefault('10.10.10.10') AND `ipv4` >= IPv4StringToNumOrDefault('9.9.9.9') ) AND NOT (isNotNull(`ipv4`))) AND (( `@timestampDateTime` <= toDateTime64(1698057560000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698056660000/1000,3) )) GROUP BY `2_@timestampDateTime` ORDER BY `2_@timestampDateTime` asc LIMIT 30000\",\n" + + " \"SELECT * FROM `table1_all` PREWHERE ((`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND ((`ipv4` = IPv4StringToNumOrDefault('3.3.3.3')) OR (`ipv4` = IPv4StringToNumOrDefault('4.4.4.4'))) AND ( `ipv4` < IPv4StringToNumOrDefault('8.8.8.8') AND `ipv4` >= IPv4StringToNumOrDefault('7.7.7.7') ) AND (isNotNull(`ipv6`))) AND ( NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND NOT ((`ipv4` = IPv4StringToNumOrDefault('5.5.5.5')) OR (`ipv4` = IPv4StringToNumOrDefault('6.6.6.6'))) AND NOT ( `ipv4` < IPv4StringToNumOrDefault('10.10.10.10') AND `ipv4` >= IPv4StringToNumOrDefault('9.9.9.9') ) AND NOT (isNotNull(`ipv4`))) AND (( `@timestampDateTime` <= toDateTime64(1698057560000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698056660000/1000,3) )) ORDER BY `@timestamp` DESC LIMIT 500\"\n" + " ]"; doTest(TEST_DATETIME64_DSL, query, Boolean.TRUE, expectedSqls); diff --git a/src/test/java/com/ly/ckibana/converter/CommonTermsAggOrderypeTest.java b/src/test/java/com/ly/ckibana/converter/CommonTermsAggOrderypeTest.java index f20a76b..5ad0e87 100644 --- a/src/test/java/com/ly/ckibana/converter/CommonTermsAggOrderypeTest.java +++ b/src/test/java/com/ly/ckibana/converter/CommonTermsAggOrderypeTest.java @@ -45,7 +45,7 @@ public class CommonTermsAggOrderypeTest extends CommonTest { public void testTermsAggOrderTypeMetricCountDesc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"_count\":\"desc\"}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890081722,\"lte\":1697890981722,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697890980000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890080000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`3__ckCount` desc LIMIT 30000\"\n" + + " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697890980000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890080000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`3__ckCount` desc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_METRIC_COUNT_DESC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -57,7 +57,7 @@ public void testTermsAggOrderTypeMetricCountDesc() { public void testTermsAggOrderTypeMetricCountAsc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"_count\":\"asc\"}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890185768,\"lte\":1697891085768,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891080000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890180000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`3__ckCount` asc LIMIT 30000\"\n" + + " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891080000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890180000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`3__ckCount` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_METRIC_COUNT_ASC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -69,7 +69,7 @@ public void testTermsAggOrderTypeMetricCountAsc() { public void testTermsAggOrderTypeMetricCustomDesc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"1\":\"desc\"}},\"aggs\":{\"1\":{\"avg\":{\"field\":\"i1\"}}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890248187,\"lte\":1697891148187,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = " [\n" + - " \"SELECT avg(`i1`) as `1_i1`,`s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891140000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890240000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`1_i1` desc LIMIT 30000\"\n" + + " \"SELECT avg(`i1`) as `1_i1`,`s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891140000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890240000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`1_i1` desc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_METRIC_CUSTOM_DESC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -81,7 +81,7 @@ public void testTermsAggOrderTypeMetricCustomDesc() { public void testTermsAggOrderTypeMetricCustomAsc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"1\":\"asc\"}},\"aggs\":{\"1\":{\"avg\":{\"field\":\"i1\"}}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890248187,\"lte\":1697891148187,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = " [\n" + - " \"SELECT avg(`i1`) as `1_i1`,`s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891140000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890240000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`1_i1` asc LIMIT 30000\"\n" + + " \"SELECT avg(`i1`) as `1_i1`,`s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891140000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890240000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`1_i1` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_METRIC_CUSTOM_ASC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -92,7 +92,7 @@ public void testTermsAggOrderTypeMetricCustomAsc() { public void testTermsAggOrderTypeAlphabetcalDesc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"_key\":\"desc\"}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697769023230,\"lte\":1697769923230,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697769920000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697769020000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`s1` desc LIMIT 30000\"\n" + + " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697769920000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697769020000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`s1` desc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_ALPHABETICAL_DESC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -103,7 +103,7 @@ public void testTermsAggOrderTypeAlphabetcalDesc() { public void testTermsAggOrderTypeAlphabetcalAsc() { String query = "{\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestampDateTime\",\"interval\":\"30s\",\"time_zone\":\"Asia/Shanghai\",\"min_doc_count\":1},\"aggs\":{\"3\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"_key\":\"asc\"}}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697890382360,\"lte\":1697891282360,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891280000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697890380000/1000.0) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`s1` asc LIMIT 30000\"\n" + + " \"SELECT `s1` as `3_s1`,count(1) as `3__ckCount`,toInt64((toUnixTimestamp64Milli(`@timestampDateTime`)) / 30000) as `2_@timestampDateTime`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1697891280000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697890380000/1000,3) ) AND (`3_s1` is not null AND `3_s1` != '' ) GROUP BY `2_@timestampDateTime`,`3_s1` ORDER BY `2_@timestampDateTime` asc,`s1` asc LIMIT 30000\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_ALPHABETICAL_ASC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } @@ -114,7 +114,7 @@ public void testTermsAggOrderTypeAlphabetcalAsc() { public void testTermsAggOrderTypeMetricCustomAscOnly() { String query = "{\"aggs\":{\"2\":{\"terms\":{\"field\":\"s1\",\"size\":5,\"order\":{\"1\":\"desc\"}},\"aggs\":{\"1\":{\"sum\":{\"field\":\"i1\"}}}}},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"match_all\":{}},{\"match_all\":{}},{\"range\":{\"@timestampDateTime\":{\"gte\":1699446469750,\"lte\":1699447369750,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"300000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT SUM(`i1`) as `1_i1`,`s1` as `2_s1`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1699447360000/1000.0) AND `@timestampDateTime` >= toDateTime64(1699446460000/1000.0) ) AND (`2_s1` is not null AND `2_s1` != '' ) GROUP BY `2_s1` ORDER BY `1_i1` desc LIMIT 5\"\n" + + " \"SELECT SUM(`i1`) as `1_i1`,`s1` as `2_s1`,count(1) as `2__ckCount` FROM `table1_all` PREWHERE ( `@timestampDateTime` <= toDateTime64(1699447360000/1000,3) AND `@timestampDateTime` >= toDateTime64(1699446460000/1000,3) ) AND (`2_s1` is not null AND `2_s1` != '' ) GROUP BY `2_s1` ORDER BY `1_i1` desc LIMIT 5\"\n" + " ]"; doTest(TEST_DEMO_TERMS_AGG_ORDER_BY_ALPHABETICAL_ASC_BASE_ON_DATE_HISTOGRAM, query, Boolean.FALSE, expectedSqls); } diff --git a/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java b/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java index e28e90e..d8fc86a 100644 --- a/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java +++ b/src/test/java/com/ly/ckibana/converter/QueryStringClauseTest.java @@ -44,7 +44,7 @@ public class QueryStringClauseTest extends CommonTest { public void testStringField() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1:\\\"s1value%\\\" AND NOT s2:\\\"%s2value\\\" AND (s2:\\\"s2value1\\\" OR s2:\\\"s2value2\\\") AND s3:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1:\\\"s1value%\\\" AND NOT s2:\\\"%s2value\\\" AND (s2:\\\"s2value1\\\" OR s2:\\\"s2value2\\\") AND s3:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697879605153,\"lte\":1697880505153,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like 's1value%' AND NOT `s2` like '%s2value' AND ( `s2` like '%s2value1%' OR `s2` like '%s2value2%') AND (`s3` like '%s3value1%' OR `s3` like '%s3value2%') AND NOT (`s3` like '%s3value3%' OR `s3` like '%s3value4%') ) AND ( `s1` like 's1value%' AND NOT `s2` like '%s2value' AND ( `s2` like '%s2value1%' OR `s2` like '%s2value2%') AND (`s3` like '%s3value1%' OR `s3` like '%s3value2%') AND NOT (`s3` like '%s3value3%' OR `s3` like '%s3value4%') ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880500000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697879600000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like 's1value%' AND NOT `s2` like '%s2value' AND ( `s2` like '%s2value1%' OR `s2` like '%s2value2%') AND (`s3` like '%s3value1%' OR `s3` like '%s3value2%') AND NOT (`s3` like '%s3value3%' OR `s3` like '%s3value4%') ) AND ( `s1` like 's1value%' AND NOT `s2` like '%s2value' AND ( `s2` like '%s2value1%' OR `s2` like '%s2value2%') AND (`s3` like '%s3value1%' OR `s3` like '%s3value2%') AND NOT (`s3` like '%s3value3%' OR `s3` like '%s3value4%') ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880500000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697879600000/1000,3) ))\"\n" + " ]"; doTest(TEST_STRING_FIELD, query, Boolean.FALSE, expectedSqls); } @@ -58,7 +58,7 @@ public void testStringField() { public void testStringFieldKeyword() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1.keyword:\\\"s1value\\\" AND NOT s2.keyword:\\\"s2value\\\" AND (s2.keyword:\\\"s2value1\\\" OR s2.keyword:\\\"s2value2\\\") AND s3.keyword:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3.keyword:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1.keyword:\\\"s1value\\\" AND NOT s2.keyword:\\\"s2value\\\" AND (s2.keyword:\\\"s2value1\\\" OR s2.keyword:\\\"s2value2\\\") AND s3.keyword:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3.keyword:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697879515585,\"lte\":1697880415585,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (`s1` = 's1value') AND NOT (`s2` = 's2value') AND ( (`s2` = 's2value1') OR (`s2` = 's2value2')) AND ((`s3` = 's3value1') OR (`s3` = 's3value2')) AND NOT ((`s3` = 's3value3') OR (`s3` = 's3value4')) ) AND ( (`s1` = 's1value') AND NOT (`s2` = 's2value') AND ( (`s2` = 's2value1') OR (`s2` = 's2value2')) AND ((`s3` = 's3value1') OR (`s3` = 's3value2')) AND NOT ((`s3` = 's3value3') OR (`s3` = 's3value4')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880410000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697879510000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (`s1` = 's1value') AND NOT (`s2` = 's2value') AND ( (`s2` = 's2value1') OR (`s2` = 's2value2')) AND ((`s3` = 's3value1') OR (`s3` = 's3value2')) AND NOT ((`s3` = 's3value3') OR (`s3` = 's3value4')) ) AND ( (`s1` = 's1value') AND NOT (`s2` = 's2value') AND ( (`s2` = 's2value1') OR (`s2` = 's2value2')) AND ((`s3` = 's3value1') OR (`s3` = 's3value2')) AND NOT ((`s3` = 's3value3') OR (`s3` = 's3value4')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880410000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697879510000/1000,3) ))\"\n" + " ]"; doTest(TEST_STRING_FIELD_KEYWORD, query, Boolean.FALSE, expectedSqls); } @@ -72,7 +72,7 @@ public void testStringFieldKeyword() { public void testStringFieldCaseIgnore() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1.caseIgnore:\\\"s1value\\\" AND NOT s2.caseIgnore:\\\"s2value\\\" AND (s2.caseIgnore:\\\"s2value1\\\" OR s2.caseIgnore:\\\"s2value2\\\") AND s3.caseIgnore:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3.caseIgnore:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1.caseIgnore:\\\"s1value\\\" AND NOT s2.caseIgnore:\\\"s2value\\\" AND (s2.caseIgnore:\\\"s2value1\\\" OR s2.caseIgnore:\\\"s2value2\\\") AND s3.caseIgnore:(\\\"s3value1\\\",\\\"s3value2\\\") AND NOT s3.caseIgnore:(\\\"s3value3\\\",\\\"s3value4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697879405576,\"lte\":1697880305576,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (positionCaseInsensitive(`s1`, 's1value') != 0) AND NOT (positionCaseInsensitive(`s2`, 's2value') != 0) AND ( (positionCaseInsensitive(`s2`, 's2value1') != 0) OR (positionCaseInsensitive(`s2`, 's2value2') != 0)) AND ((positionCaseInsensitive(`s3`, 's3value1') != 0) OR (positionCaseInsensitive(`s3`, 's3value2') != 0)) AND NOT ((positionCaseInsensitive(`s3`, 's3value3') != 0) OR (positionCaseInsensitive(`s3`, 's3value4') != 0)) ) AND ( (positionCaseInsensitive(`s1`, 's1value') != 0) AND NOT (positionCaseInsensitive(`s2`, 's2value') != 0) AND ( (positionCaseInsensitive(`s2`, 's2value1') != 0) OR (positionCaseInsensitive(`s2`, 's2value2') != 0)) AND ((positionCaseInsensitive(`s3`, 's3value1') != 0) OR (positionCaseInsensitive(`s3`, 's3value2') != 0)) AND NOT ((positionCaseInsensitive(`s3`, 's3value3') != 0) OR (positionCaseInsensitive(`s3`, 's3value4') != 0)) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880300000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697879400000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (positionCaseInsensitive(`s1`, 's1value') != 0) AND NOT (positionCaseInsensitive(`s2`, 's2value') != 0) AND ( (positionCaseInsensitive(`s2`, 's2value1') != 0) OR (positionCaseInsensitive(`s2`, 's2value2') != 0)) AND ((positionCaseInsensitive(`s3`, 's3value1') != 0) OR (positionCaseInsensitive(`s3`, 's3value2') != 0)) AND NOT ((positionCaseInsensitive(`s3`, 's3value3') != 0) OR (positionCaseInsensitive(`s3`, 's3value4') != 0)) ) AND ( (positionCaseInsensitive(`s1`, 's1value') != 0) AND NOT (positionCaseInsensitive(`s2`, 's2value') != 0) AND ( (positionCaseInsensitive(`s2`, 's2value1') != 0) OR (positionCaseInsensitive(`s2`, 's2value2') != 0)) AND ((positionCaseInsensitive(`s3`, 's3value1') != 0) OR (positionCaseInsensitive(`s3`, 's3value2') != 0)) AND NOT ((positionCaseInsensitive(`s3`, 's3value3') != 0) OR (positionCaseInsensitive(`s3`, 's3value4') != 0)) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880300000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697879400000/1000,3) ))\"\n" + " ]"; doTest(TEST_STRING_FIELD_IGNORE_CASE, query, Boolean.FALSE, expectedSqls); } @@ -86,7 +86,7 @@ public void testStringFieldCaseIgnore() { public void testStringFieldCaseIgnoreKeyword() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1.caseIgnore.keyword:\\\"s1VALUe\\\" AND NOT s2.caseIgnore.keyword:\\\"s2VALUe\\\" AND (s2.caseIgnore.keyword:\\\"s2VALUe1\\\" OR s2.caseIgnore.keyword:\\\"s2VALUe2\\\") AND s3.caseIgnore.keyword:(\\\"s3VALUe1\\\",\\\"s3VALUe2\\\") AND NOT s3.caseIgnore.keyword:(\\\"s3VALUe3\\\",\\\"s3VALUe4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1.caseIgnore.keyword:\\\"s1VALUe\\\" AND NOT s2.caseIgnore.keyword:\\\"s2VALUe\\\" AND (s2.caseIgnore.keyword:\\\"s2VALUe1\\\" OR s2.caseIgnore.keyword:\\\"s2VALUe2\\\") AND s3.caseIgnore.keyword:(\\\"s3VALUe1\\\",\\\"s3VALUe2\\\") AND NOT s3.caseIgnore.keyword:(\\\"s3VALUe3\\\",\\\"s3VALUe4\\\")\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697879117903,\"lte\":1697880017903,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (lower('s1') = 's1value') AND NOT (lower('s2') = 's2value') AND ( (lower('s2') = 's2value1') OR (lower('s2') = 's2value2')) AND ((lower('`s3`') = 's3value1') OR (lower('`s3`') = 's3value2')) AND NOT ((lower('`s3`') = 's3value3') OR (lower('`s3`') = 's3value4')) ) AND ( (lower('s1') = 's1value') AND NOT (lower('s2') = 's2value') AND ( (lower('s2') = 's2value1') OR (lower('s2') = 's2value2')) AND ((lower('`s3`') = 's3value1') OR (lower('`s3`') = 's3value2')) AND NOT ((lower('`s3`') = 's3value3') OR (lower('`s3`') = 's3value4')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880010000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697879110000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( (lower('s1') = 's1value') AND NOT (lower('s2') = 's2value') AND ( (lower('s2') = 's2value1') OR (lower('s2') = 's2value2')) AND ((lower('`s3`') = 's3value1') OR (lower('`s3`') = 's3value2')) AND NOT ((lower('`s3`') = 's3value3') OR (lower('`s3`') = 's3value4')) ) AND ( (lower('s1') = 's1value') AND NOT (lower('s2') = 's2value') AND ( (lower('s2') = 's2value1') OR (lower('s2') = 's2value2')) AND ((lower('`s3`') = 's3value1') OR (lower('`s3`') = 's3value2')) AND NOT ((lower('`s3`') = 's3value3') OR (lower('`s3`') = 's3value4')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697880010000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697879110000/1000,3) ))\"\n" + " ]"; doTest(TEST_STRING_FIELD_KEYWORD_IGNORE_CASE, query, Boolean.FALSE, expectedSqls); } @@ -100,7 +100,7 @@ public void testStringFieldCaseIgnoreKeyword() { public void testIpWithStringType() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ip:\\\"1.1.1.1\\\" AND NOT ip:\\\"2.2.2.2\\\" AND ip:[\\\"3.3.3.3\\\" TO \\\"4.4.4.4\\\"] AND NOT ip:[\\\"5.5.5.5\\\" TO \\\"6.6.6.6\\\"]) OR (ip:(\\\"7.7.7.7\\\",\\\"8.8.8.8\\\") AND ip:(\\\"9.9.9.9\\\",\\\"10.10.10.10\\\") ) AND (ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"]) OR ( ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] ) AND ip:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\") AND NOT ( ip:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ip:\\\"1.1.1.1\\\" AND NOT ip:\\\"2.2.2.2\\\" AND ip:[\\\"3.3.3.3\\\" TO \\\"4.4.4.4\\\"] AND NOT ip:[\\\"5.5.5.5\\\" TO \\\"6.6.6.6\\\"]) OR (ip:(\\\"7.7.7.7\\\",\\\"8.8.8.8\\\") AND ip:(\\\"9.9.9.9\\\",\\\"10.10.10.10\\\") ) AND (ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"]) OR ( ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ip:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ip:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] ) AND ip:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\") AND NOT ( ip:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1698054551914,\"lte\":1698055451914,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( `ip` like '%1.1.1.1%' AND NOT `ip` like '%2.2.2.2%' AND ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('3.3.3.3') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('4.4.4.4')) AND NOT ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('5.5.5.5') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('6.6.6.6'))) OR ( (`ip` like '%7.7.7.7%' OR `ip` like '%8.8.8.8%') AND (`ip` like '%9.9.9.9%' OR `ip` like '%10.10.10.10%')) AND ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%') AND NOT ( (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%')) ) AND ( `s1` like '%s1value1%' AND ( `ip` like '%1.1.1.1%' AND NOT `ip` like '%2.2.2.2%' AND ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('3.3.3.3') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('4.4.4.4')) AND NOT ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('5.5.5.5') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('6.6.6.6'))) OR ( (`ip` like '%7.7.7.7%' OR `ip` like '%8.8.8.8%') AND (`ip` like '%9.9.9.9%' OR `ip` like '%10.10.10.10%')) AND ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%') AND NOT ( (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055450000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698054550000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( `ip` like '%1.1.1.1%' AND NOT `ip` like '%2.2.2.2%' AND ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('3.3.3.3') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('4.4.4.4')) AND NOT ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('5.5.5.5') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('6.6.6.6'))) OR ( (`ip` like '%7.7.7.7%' OR `ip` like '%8.8.8.8%') AND (`ip` like '%9.9.9.9%' OR `ip` like '%10.10.10.10%')) AND ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%') AND NOT ( (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%')) ) AND ( `s1` like '%s1value1%' AND ( `ip` like '%1.1.1.1%' AND NOT `ip` like '%2.2.2.2%' AND ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('3.3.3.3') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('4.4.4.4')) AND NOT ( IPv4StringToNumOrDefault(`ip`) >= IPv4StringToNum('5.5.5.5') AND IPv4StringToNumOrDefault(`ip`) <= IPv4StringToNum('6.6.6.6'))) OR ( (`ip` like '%7.7.7.7%' OR `ip` like '%8.8.8.8%') AND (`ip` like '%9.9.9.9%' OR `ip` like '%10.10.10.10%')) AND ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND NOT `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' AND ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( IPv6StringToNumOrDefault(`ip`) >= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND IPv6StringToNumOrDefault(`ip`) <= IPv6StringToNum('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%') AND NOT ( (`ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%' OR `ip` like '%2001:0db8:85a3:0000:0000:8a2e:0370:7334%')) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055450000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698054550000/1000,3) ))\"\n" + " ]"; doTest(TEST_IP_TYPE_STRING, query, Boolean.FALSE, expectedSqls); } @@ -114,7 +114,7 @@ public void testIpWithStringType() { public void testIpWithIpv4Type() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ipv4:\\\"1.1.1.1\\\" AND NOT ipv4:\\\"2.2.2.2\\\" AND ipv4:[\\\"3.3.3.3\\\" TO \\\"4.4.4.4\\\"] AND NOT ipv4:[\\\"5.5.5.5\\\" TO \\\"6.6.6.6\\\"]) OR ( ipv4:(\\\"7.7.7.7\\\",\\\"8.8.8.8\\\") AND NOT ipv4:(\\\"9.9.9.9.9\\\",\\\"10.10.10.10\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ipv4:\\\"1.1.1.1\\\" AND NOT ipv4:\\\"2.2.2.2\\\" AND ipv4:[\\\"3.3.3.3\\\" TO \\\"4.4.4.4\\\"] AND NOT ipv4:[\\\"5.5.5.5\\\" TO \\\"6.6.6.6\\\"]) OR ( ipv4:(\\\"7.7.7.7\\\",\\\"8.8.8.8\\\") AND NOT ipv4:(\\\"9.9.9.9.9\\\",\\\"10.10.10.10\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1698054288267,\"lte\":1698055188267,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( (`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND ( `ipv4` >= IPv4StringToNumOrDefault('3.3.3.3') AND `ipv4` <= IPv4StringToNumOrDefault('4.4.4.4')) AND NOT ( `ipv4` >= IPv4StringToNumOrDefault('5.5.5.5') AND `ipv4` <= IPv4StringToNumOrDefault('6.6.6.6'))) OR ( `ipv4` in (IPv4StringToNumOrDefault('7.7.7.7'),IPv4StringToNumOrDefault('8.8.8.8')) AND NOT `ipv4` in (IPv4StringToNumOrDefault('9.9.9.9.9'),IPv4StringToNumOrDefault('10.10.10.10'))) ) AND ( `s1` like '%s1value1%' AND ( (`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND ( `ipv4` >= IPv4StringToNumOrDefault('3.3.3.3') AND `ipv4` <= IPv4StringToNumOrDefault('4.4.4.4')) AND NOT ( `ipv4` >= IPv4StringToNumOrDefault('5.5.5.5') AND `ipv4` <= IPv4StringToNumOrDefault('6.6.6.6'))) OR ( `ipv4` in (IPv4StringToNumOrDefault('7.7.7.7'),IPv4StringToNumOrDefault('8.8.8.8')) AND NOT `ipv4` in (IPv4StringToNumOrDefault('9.9.9.9.9'),IPv4StringToNumOrDefault('10.10.10.10'))) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055180000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698054280000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( (`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND ( `ipv4` >= IPv4StringToNumOrDefault('3.3.3.3') AND `ipv4` <= IPv4StringToNumOrDefault('4.4.4.4')) AND NOT ( `ipv4` >= IPv4StringToNumOrDefault('5.5.5.5') AND `ipv4` <= IPv4StringToNumOrDefault('6.6.6.6'))) OR ( `ipv4` in (IPv4StringToNumOrDefault('7.7.7.7'),IPv4StringToNumOrDefault('8.8.8.8')) AND NOT `ipv4` in (IPv4StringToNumOrDefault('9.9.9.9.9'),IPv4StringToNumOrDefault('10.10.10.10'))) ) AND ( `s1` like '%s1value1%' AND ( (`ipv4` = IPv4StringToNumOrDefault('1.1.1.1')) AND NOT (`ipv4` = IPv4StringToNumOrDefault('2.2.2.2')) AND ( `ipv4` >= IPv4StringToNumOrDefault('3.3.3.3') AND `ipv4` <= IPv4StringToNumOrDefault('4.4.4.4')) AND NOT ( `ipv4` >= IPv4StringToNumOrDefault('5.5.5.5') AND `ipv4` <= IPv4StringToNumOrDefault('6.6.6.6'))) OR ( `ipv4` in (IPv4StringToNumOrDefault('7.7.7.7'),IPv4StringToNumOrDefault('8.8.8.8')) AND NOT `ipv4` in (IPv4StringToNumOrDefault('9.9.9.9.9'),IPv4StringToNumOrDefault('10.10.10.10'))) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055180000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698054280000/1000,3) ))\"\n" + " ]"; doTest(TEST_IP_TYPE_IPV4, query, Boolean.FALSE, expectedSqls); } @@ -128,7 +128,7 @@ public void testIpWithIpv4Type() { public void testIpWithIpv6Type() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"]) OR ( ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] ) AND ipv6:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\") AND NOT ( ipv6:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"s1:\\\"s1value1\\\" AND (ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"]) OR ( ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND NOT ipv6:\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" AND ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] AND NOT ipv6:[\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\" TO \\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"] ) AND ipv6:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\") AND NOT ( ipv6:(\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\",\\\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\\\"))\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1698054706846,\"lte\":1698055606846,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) ) AND ( `s1` like '%s1value1%' AND ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055600000/1000.0) AND `@timestampDateTime` >= toDateTime64(1698054700000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `s1` like '%s1value1%' AND ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) ) AND ( `s1` like '%s1value1%' AND ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) OR ( (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT (`ipv6` = IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` >= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334') AND `ipv6` <= IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) AND `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334')) AND NOT ( `ipv6` in (IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'),IPv6StringToNumOrDefault('2001:0db8:85a3:0000:0000:8a2e:0370:7334'))) ) ) AND (( `@timestampDateTime` <= toDateTime64(1698055600000/1000,3) AND `@timestampDateTime` >= toDateTime64(1698054700000/1000,3) ))\"\n" + " ]"; doTest(TEST_IP_TYPE_IPV6, query, Boolean.FALSE, expectedSqls); } @@ -142,7 +142,7 @@ public void testIpWithIpv6Type() { public void testNumberField() { String query = "{\"aggs\":{},\"size\":0,\"_source\":{\"excludes\":[]},\"stored_fields\":[\"*\"],\"script_fields\":{},\"docvalue_fields\":[{\"field\":\"@timestampDateTime\",\"format\":\"date_time\"}],\"query\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"i1:1 AND NOT i1:2 AND i1:[2 TO 3] AND NOT i1:[4 TO 5] AND i1:(6,7) AND NOT i1:(8,9) AND (i2:>10 AND i2:>=11 OR (i2:<10 AND i2:<=11)) AND (NOT i3:>12 AND NOT i3:>=13 AND NOT i3:<14 AND NOT i2:<=15)\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"query_string\":{\"query\":\"i1:1 AND NOT i1:2 AND i1:[2 TO 3] AND NOT i1:[4 TO 5] AND i1:(6,7) AND NOT i1:(8,9) AND (i2:>10 AND i2:>=11 OR (i2:<10 AND i2:<=11)) AND (NOT i3:>12 AND NOT i3:>=13 AND NOT i3:<14 AND NOT i2:<=15)\",\"analyze_wildcard\":true,\"default_field\":\"*\"}},{\"range\":{\"@timestampDateTime\":{\"gte\":1697889668151,\"lte\":1697890568151,\"format\":\"epoch_millis\"}}}],\"filter\":[],\"should\":[],\"must_not\":[]}},\"timeout\":\"120000ms\"}"; String expectedSqls = "[\n" + - " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `i1` = 1 AND NOT `i1` = 2 AND ( `i1` >= 2 AND `i1` <= 3) AND NOT ( `i1` >= 4 AND `i1` <= 5) AND `i1` in (6,7) AND NOT `i1` in (8,9) AND ( `i2` >10 AND `i2` >=11 OR ( `i2` <10 AND `i2` <=11)) AND ( NOT `i3` >12 AND NOT `i3` >=13 AND NOT `i3` <14 AND NOT `i2` <=15) ) AND ( `i1` = 1 AND NOT `i1` = 2 AND ( `i1` >= 2 AND `i1` <= 3) AND NOT ( `i1` >= 4 AND `i1` <= 5) AND `i1` in (6,7) AND NOT `i1` in (8,9) AND ( `i2` >10 AND `i2` >=11 OR ( `i2` <10 AND `i2` <=11)) AND ( NOT `i3` >12 AND NOT `i3` >=13 AND NOT `i3` <14 AND NOT `i2` <=15) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697890560000/1000.0) AND `@timestampDateTime` >= toDateTime64(1697889660000/1000.0) ))\"\n" + + " \"SELECT count(1) as _count FROM `table1_all` PREWHERE ( ( `i1` = 1 AND NOT `i1` = 2 AND ( `i1` >= 2 AND `i1` <= 3) AND NOT ( `i1` >= 4 AND `i1` <= 5) AND `i1` in (6,7) AND NOT `i1` in (8,9) AND ( `i2` >10 AND `i2` >=11 OR ( `i2` <10 AND `i2` <=11)) AND ( NOT `i3` >12 AND NOT `i3` >=13 AND NOT `i3` <14 AND NOT `i2` <=15) ) AND ( `i1` = 1 AND NOT `i1` = 2 AND ( `i1` >= 2 AND `i1` <= 3) AND NOT ( `i1` >= 4 AND `i1` <= 5) AND `i1` in (6,7) AND NOT `i1` in (8,9) AND ( `i2` >10 AND `i2` >=11 OR ( `i2` <10 AND `i2` <=11)) AND ( NOT `i3` >12 AND NOT `i3` >=13 AND NOT `i3` <14 AND NOT `i2` <=15) ) ) AND (( `@timestampDateTime` <= toDateTime64(1697890560000/1000,3) AND `@timestampDateTime` >= toDateTime64(1697889660000/1000,3) ))\"\n" + " ]"; doTest(TEST_NUMBER, query, Boolean.FALSE, expectedSqls); } From 732bb369fa33e2c2b391330d344f825e870c7f80 Mon Sep 17 00:00:00 2001 From: caojiaqiang Date: Mon, 29 Jan 2024 15:48:42 +0800 Subject: [PATCH 2/2] feat: code format --- .../java/com/ly/ckibana/handlers/ParamHandler.java | 2 +- src/main/java/com/ly/ckibana/util/DateUtils.java | 1 - src/main/java/com/ly/ckibana/util/ProxyUtils.java | 2 +- src/main/java/com/ly/ckibana/util/RestUtils.java | 2 +- src/main/java/com/ly/ckibana/util/SqlUtils.java | 10 ++++------ 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/ly/ckibana/handlers/ParamHandler.java b/src/main/java/com/ly/ckibana/handlers/ParamHandler.java index 2be7fbc..088fe7b 100644 --- a/src/main/java/com/ly/ckibana/handlers/ParamHandler.java +++ b/src/main/java/com/ly/ckibana/handlers/ParamHandler.java @@ -140,7 +140,7 @@ public String doHandle(RequestContext context) { } String body = proxyConfigLoader.getYaml().dumpAs(kibanaProperty, Tag.MAP, DumperOptions.FlowStyle.BLOCK); - String response = EsClientUtil.saveOne(proxyConfigLoader.getMetadataRestClient(), proxyConfigLoader.getSettingsIndexName(), + String response = EsClientUtil.saveOne(proxyConfigLoader.getMetadataRestClient(), proxyConfigLoader.getSettingsIndexName(), "kibana", Map.of( "key", "kibana", diff --git a/src/main/java/com/ly/ckibana/util/DateUtils.java b/src/main/java/com/ly/ckibana/util/DateUtils.java index 4a0427e..86b102b 100644 --- a/src/main/java/com/ly/ckibana/util/DateUtils.java +++ b/src/main/java/com/ly/ckibana/util/DateUtils.java @@ -29,7 +29,6 @@ import java.util.Date; import java.util.List; import java.util.Locale; -import java.sql.Timestamp; @Slf4j public class DateUtils { diff --git a/src/main/java/com/ly/ckibana/util/ProxyUtils.java b/src/main/java/com/ly/ckibana/util/ProxyUtils.java index 0d285e6..dc9485a 100644 --- a/src/main/java/com/ly/ckibana/util/ProxyUtils.java +++ b/src/main/java/com/ly/ckibana/util/ProxyUtils.java @@ -279,7 +279,7 @@ public static Range getRangeWrappedBySqlFunction(Range orgRange, boolean isTimeF */ public static String generateTimeFieldSqlWithFormatDateTime64ZoneShangHai(Object value, String ckFieldType) { if (SqlUtils.isDateTime64(ckFieldType)) { - return String.format("toDateTime64(%s/1000,%d)", value.toString(),SqlUtils.getDateTime64Scale(ckFieldType)); + return String.format("toDateTime64(%s/1000,%d)", value.toString(), SqlUtils.getDateTime64Scale(ckFieldType)); } else if (SqlUtils.isDateTime(ckFieldType)) { return String.format("toDateTime(%s/1000)", value.toString()); } else { diff --git a/src/main/java/com/ly/ckibana/util/RestUtils.java b/src/main/java/com/ly/ckibana/util/RestUtils.java index 6f7b15a..066dbe5 100644 --- a/src/main/java/com/ly/ckibana/util/RestUtils.java +++ b/src/main/java/com/ly/ckibana/util/RestUtils.java @@ -87,7 +87,7 @@ public static RestClient initEsRestClient(String host, Map heade if (each.contains(":")) { String[] splits = each.split(":"); hosts[i] = new HttpHost(splits[0], Integer.parseInt(splits[1])); - }else { + } else { hosts[i] = new HttpHost(host); } } diff --git a/src/main/java/com/ly/ckibana/util/SqlUtils.java b/src/main/java/com/ly/ckibana/util/SqlUtils.java index 1949923..73c4f63 100644 --- a/src/main/java/com/ly/ckibana/util/SqlUtils.java +++ b/src/main/java/com/ly/ckibana/util/SqlUtils.java @@ -407,7 +407,6 @@ public static String getSelectTemplate(String selectSql, String table) { return String.format("SELECT %s FROM %s", selectSql, table); } - /** * 是否为DateTime64时间类型. * @@ -418,17 +417,16 @@ public static boolean isDateTime64(String ckFieldType) { return StringUtils.startsWith(ckFieldType, SqlConstants.TYPE_DATETIME64); } - /** - * DateTime64时间类型，提取精度 + * DateTime64时间类型，提取精度. * * @param ckFieldType ck字段类型,如DateTime64(3) * @return 精度,如3 */ public static int getDateTime64Scale(String ckFieldType) { - String scale= ckFieldType.replace(SqlConstants.TYPE_DATETIME64,StringUtils.EMPTY) - .replace(Constants.Symbol.LEFT_PARENTHESIS,StringUtils.EMPTY) - .replace(Constants.Symbol.RIGHT_PARENTHESIS,StringUtils.EMPTY); + String scale = ckFieldType.replace(SqlConstants.TYPE_DATETIME64, StringUtils.EMPTY) + .replace(Constants.Symbol.LEFT_PARENTHESIS, StringUtils.EMPTY) + .replace(Constants.Symbol.RIGHT_PARENTHESIS, StringUtils.EMPTY); return Integer.parseInt(scale); } /**