From e3efa65c1cb91159cf6656222d232f87c707eb98 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Mon, 4 Nov 2024 14:38:18 -0800 Subject: [PATCH 01/21] add floor and ceil func --- .../filter/TimePredicateFilterOptimizer.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 2fcba071677..c868f3aff24 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.common.function.DateTimeUtils; import org.apache.pinot.common.request.Expression; import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.Function; @@ -38,6 +39,7 @@ import org.apache.pinot.spi.data.Schema; import org.apache.pinot.spi.utils.TimeUtils; import org.apache.pinot.sql.FilterKind; +import org.joda.time.chrono.ISOChronology; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -411,6 +413,10 @@ && isStringLiteral(dateTimeConvertOperands.get(3)), } } + private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { + + } + private boolean isStringLiteral(Expression expression) { Literal literal = expression.getLiteral(); return literal != null && literal.isSetStringValue(); @@ -446,4 +452,18 @@ private static void rewriteToRange(Function filterFunction, Expression expressio newOperands.add(RequestUtils.getLiteralExpression(rangeString)); filterFunction.setOperands(newOperands); } + + private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, + String outputTimeUnit) { + return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) + .roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))), + TimeUnit.MILLISECONDS); + } + + private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, + String outputTimeUnit) { + return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) + .roundCeiling(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))), + TimeUnit.MILLISECONDS); + } } From 749bbbb2d99a5174ad04c19b4bfe8441e6de3e47 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Mon, 4 Nov 2024 21:50:44 -0800 Subject: [PATCH 02/21] add date trunc optimization --- .../filter/TimePredicateFilterOptimizer.java | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index c868f3aff24..064aecc43cf 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -26,12 +26,14 @@ import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.function.DateTimeUtils; +import org.apache.pinot.common.function.TimeZoneKey; import org.apache.pinot.common.request.Expression; import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.Function; import org.apache.pinot.common.request.Literal; import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.core.operator.transform.function.DateTimeConversionTransformFunction; +import org.apache.pinot.core.operator.transform.function.DateTruncTransformFunction; import org.apache.pinot.core.operator.transform.function.TimeConversionTransformFunction; import org.apache.pinot.spi.data.DateTimeFieldSpec.TimeFormat; import org.apache.pinot.spi.data.DateTimeFormatSpec; @@ -86,6 +88,8 @@ Expression optimize(Expression filterExpression) { optimizeTimeConvert(filterFunction, filterKind); } else if (functionName.equalsIgnoreCase(DateTimeConversionTransformFunction.FUNCTION_NAME)) { optimizeDateTimeConvert(filterFunction, filterKind); + } else if (functionName.equalsIgnoreCase(DateTruncTransformFunction.FUNCTION_NAME)) { + optimizeDateTrunc(filterFunction, filterKind); } } } @@ -414,7 +418,95 @@ && isStringLiteral(dateTimeConvertOperands.get(3)), } private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { + List filterOperands = filterFunction.getOperands(); + List dateTruncOperands = filterOperands.get(0).getFunctionCall().getOperands(); + + Long lowerMillis = null; + Long upperMillis = null; + DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); + boolean lowerInclusive = true; + boolean upperInclusive = true; + + switch (filterKind) { + case EQUALS: + if (dateTruncOperands.size() == 2) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 4) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + } else if (dateTruncOperands.size() == 5) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) + .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) + .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + } + break; + case GREATER_THAN: + lowerInclusive = false; + case GREATER_THAN_OR_EQUAL: + if (dateTruncOperands.size() == 2) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue()); + } else if (dateTruncOperands.size() == 4) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + } else if (dateTruncOperands.size() == 5) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) + .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + } + break; + case LESS_THAN: + upperInclusive = false; + case LESS_THAN_OR_EQUAL: + if (dateTruncOperands.size() == 2) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 4) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + } else if (dateTruncOperands.size() == 5) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) + .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + } + break; + default: + throw new IllegalStateException(); + } + lowerMillis = (lowerMillis == null) ? 0 : lowerMillis; + upperMillis = (upperMillis == null) ? 0 : upperMillis; + String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); + rewriteToRange(filterFunction, dateTruncOperands.get(0), rangeString); } private boolean isStringLiteral(Expression expression) { @@ -453,6 +545,20 @@ private static void rewriteToRange(Function filterFunction, Expression expressio filterFunction.setOperands(newOperands); } + private static long dateTruncFloor(String unit, long timeValue) { + return dateTruncFloor(unit, timeValue, TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), + TimeUnit.MILLISECONDS.name()); + } + + private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit) { + return dateTruncFloor(unit, timeValue, inputTimeUnit, ISOChronology.getInstanceUTC(), inputTimeUnit); + } + + private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit, String timeZone) { + return dateTruncFloor(unit, timeValue, inputTimeUnit, + DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(timeZone)), inputTimeUnit); + } + private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, String outputTimeUnit) { return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) @@ -460,6 +566,20 @@ private static long dateTruncFloor(String unit, long timeValue, String inputTime TimeUnit.MILLISECONDS); } + private static long dateTruncCeil(String unit, long timeValue) { + return dateTruncCeil(unit, timeValue, TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), + TimeUnit.MILLISECONDS.name()); + } + + private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit) { + return dateTruncCeil(unit, timeValue, inputTimeUnit, ISOChronology.getInstanceUTC(), inputTimeUnit); + } + + private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, String timeZone) { + return dateTruncCeil(unit, timeValue, inputTimeUnit, + DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(timeZone)), inputTimeUnit); + } + private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, String outputTimeUnit) { return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) From c2dbe084b6d48af891c7b185d7fe4376f0527eb7 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Mon, 4 Nov 2024 21:55:48 -0800 Subject: [PATCH 03/21] fix style --- .../filter/TimePredicateFilterOptimizer.java | 95 ++++++++++++++----- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 064aecc43cf..40845fcb379 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -436,51 +436,96 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) + .getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() + .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) - .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils + .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), + dateTruncOperands.get(4).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) - .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands + .get(4).getLiteral().getStringValue()); } break; case GREATER_THAN: lowerInclusive = false; - case GREATER_THAN_OR_EQUAL: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue()); + } else if (dateTruncOperands.size() == 4) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); + } else if (dateTruncOperands.size() == 5) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) + .getLiteral().getStringValue()); + } + break; + case GREATER_THAN_OR_EQUAL: + if (dateTruncOperands.size() == 2) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue()); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() + .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) - .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) + .getLiteral().getStringValue()); } break; case LESS_THAN: upperInclusive = false; + if (dateTruncOperands.size() == 2) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 4) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() + .getStringValue()); + } else if (dateTruncOperands.size() == 5) { + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) + .getLiteral().getStringValue()); + } + break; case LESS_THAN_OR_EQUAL: if (dateTruncOperands.size() == 2) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), @@ -490,13 +535,15 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); } else if (dateTruncOperands.size() == 4) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), dateTruncOperands.get(3).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() + .getStringValue()); } else if (dateTruncOperands.size() == 5) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), dateTruncOperands.get(2).getLiteral() - .getStringValue(), DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3) - .getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) + .getLiteral().getStringValue()); } break; default: From 8dda15e1930045a7ae5672dd5edf1d237a7b009c Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Mon, 4 Nov 2024 22:26:14 -0800 Subject: [PATCH 04/21] fix range query column expr --- .../query/optimizer/filter/TimePredicateFilterOptimizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 40845fcb379..f7b85308d8c 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -553,7 +553,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { upperMillis = (upperMillis == null) ? 0 : upperMillis; String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); - rewriteToRange(filterFunction, dateTruncOperands.get(0), rangeString); + rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); } private boolean isStringLiteral(Expression expression) { From 2edad2a26054aebe31a371d32d12f4d6265117bc Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 11:20:32 -0800 Subject: [PATCH 05/21] add support for between --- .../filter/TimePredicateFilterOptimizer.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index f7b85308d8c..2fe3771542e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -421,6 +421,11 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { List filterOperands = filterFunction.getOperands(); List dateTruncOperands = filterOperands.get(0).getFunctionCall().getOperands(); + // Check if date trunc function is being applied on a literal value + if (dateTruncOperands.get(1).isSetLiteral()) { + return; + } + Long lowerMillis = null; Long upperMillis = null; DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); @@ -546,6 +551,41 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { .getLiteral().getStringValue()); } break; + case BETWEEN: + if (dateTruncOperands.size() == 2) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue())); + } else if (dateTruncOperands.size() == 3) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue()); + } else if (dateTruncOperands.size() == 4) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) + .getLiteral().getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() + .getStringValue()); + } else if (dateTruncOperands.size() == 5) { + lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils + .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), + dateTruncOperands.get(4).getLiteral().getStringValue()); + upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands + .get(4).getLiteral().getStringValue()); + } + break; default: throw new IllegalStateException(); } From 52490a3db75950ae829b0008fd3bc298e44514f4 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 11:33:56 -0800 Subject: [PATCH 06/21] refactor test functions --- .../TimePredicateFilterOptimizerTest.java | 98 ++++++++++--------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index 1ecb7f211ad..e63fb8ee5fa 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -40,117 +40,125 @@ public void testTimeConvert() { testNoOpTimeConvert("TIMECONVERT(col, 'MILLISECONDS', 'MILLISECONDS') = 1620830760000"); // Other output format - testTimeConvert("timeConvert(col, 'MILLISECONDS', 'SECONDS') > 1620830760", + testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'SECONDS') > 1620830760", new Range(1620830761000L, true, null, false)); - testTimeConvert("timeConvert(col, 'MILLISECONDS', 'MINUTES') < 27015286", + testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'MINUTES') < 27015286", new Range(null, false, 1620917160000L, false)); - testTimeConvert("timeConvert(col, 'MILLISECONDS', 'HOURS') BETWEEN 450230 AND 450254", + testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'HOURS') BETWEEN 450230 AND 450254", new Range(1620828000000L, true, 1620918000000L, false)); - testTimeConvert("timeConvert(col, 'MILLISECONDS', 'DAYS') = 18759", + testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'DAYS') = 18759", new Range(1620777600000L, true, 1620864000000L, false)); // Other input format - testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", new Range(27013846L, false, null, false)); - testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", new Range(null, false, 450254L, true)); - testTimeConvert("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254", + testPredicateFilterOptimizer("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", + new Range(27013846L, false, null, false)); + testPredicateFilterOptimizer("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", + new Range(null, false, 450254L, true)); + testPredicateFilterOptimizer("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254", new Range(18759L, false, 18760L, true)); - testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759", new Range(1620777600L, true, 1620864000L, false)); + testPredicateFilterOptimizer("timeConvert(col, 'SECONDS', 'DAYS') = 18759", + new Range(1620777600L, true, 1620864000L, false)); // Invalid time - testInvalidTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5"); - testInvalidTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760"); + testInvalidFilterOptimizer("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5"); + testInvalidFilterOptimizer("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760"); } @Test public void testEpochToEpochDateTimeConvert() { // Value not on granularity boundary - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620830760000", new Range(1620831600000L, true, null, false)); - testTimeConvert( + testPredicateFilterOptimizer( "DATE_TIME_CONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false, 1620918000000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "datetimeconvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760000 AND" + " 1620917160000", new Range(1620831600000L, true, 1620918000000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "DATETIMECONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620830760000", new Range(1620831600000L, true, 1620831600000L, false)); // Value on granularity boundary - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620831600000", new Range(1620833400000L, true, null, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620918000000", new Range(null, false, 1620918000000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620831600000 AND" + " 1620918000000", new Range(1620831600000L, true, 1620919800000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620831600000", new Range(1620831600000L, true, 1620833400000L, false)); // Other output format - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')" + + " > 27013846", new Range(1620831600000L, true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') " + + "< 2701528", new Range(null, false, 1620918000000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND " + "1620917160", new Range(1620831600000L, true, 1620918000000L, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') " + + "> 900462", new Range(1620833400000L, true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", new Range(null, false, 1620918000000L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760", new Range(1620777600000L, true, 1620950400000L, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", new Range(1620777600000L, true, 1620864000000L, false)); // Other input format - testTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", new Range(1620831600L, true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MINUTES:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", new Range(null, false, 27015300L, false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:DAYS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND 1620917160", new Range(18759L, false, 18760L, true)); - testTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", new Range(1620833400L, true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", new Range(null, false, 27015300L, false)); - testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') " + + "BETWEEN 18759 AND 18760", new Range(18759L, true, 18761L, false)); - testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", + testPredicateFilterOptimizer("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", new Range(18759L, true, 18760L, false)); // Invalid time - testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5"); - testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846"); + testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5"); + testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846"); } @Test public void testSDFToEpochDateTimeConvert() { - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', " + "'30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', " - + "'30:MINUTES') < 1620917160000", new Range(null, false, "2021-05-13 15:00:00", false)); - testTimeConvert( + testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', " + + "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false, + "2021-05-13 15:00:00", false)); + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', '30:MINUTES') " + "BETWEEN 1620830760000 AND 1620917160000", new Range("2021-05-12 15:00", true, "2021-05-13 15:00", false)); - testTimeConvert( + testPredicateFilterOptimizer( "dateTimeConvert(col, '1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd', '1:MILLISECONDS:EPOCH', '30:MINUTES') = " + "1620830760000", new Range("2021-05-12", false, "2021-05-12", true)); // Invalid time - testInvalidTimeConvert( + testInvalidFilterOptimizer( "dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', " + "'30:MINUTES') > 1620830760000.5"); - testInvalidTimeConvert( + testInvalidFilterOptimizer( "dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', " + "'30:MINUTES') < 1620917160"); } @@ -176,9 +184,9 @@ private void testNoOpTimeConvert(String filterString) { } /** - * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT on the given filter. + * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT/DATE_TRUNC on the given filter. */ - private void testTimeConvert(String filterString, Range expectedRange) { + private void testPredicateFilterOptimizer(String filterString, Range expectedRange) { Expression originalExpression = CalciteSqlParser.compileToExpression(filterString); Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString)); Function function = optimizedFilterExpression.getFunctionCall(); @@ -192,9 +200,9 @@ private void testTimeConvert(String filterString, Range expectedRange) { } /** - * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT with invalid time in filter. + * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT/DATE_TRUNC with invalid time in filter. */ - private void testInvalidTimeConvert(String filterString) { + private void testInvalidFilterOptimizer(String filterString) { Expression originalExpression = CalciteSqlParser.compileToExpression(filterString); Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString)); assertEquals(optimizedFilterExpression, originalExpression); From cb387da9eea0c9193fade2347802b09d3716d8d8 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 12:32:05 -0800 Subject: [PATCH 07/21] refactor back and add test --- .../filter/TimePredicateFilterOptimizer.java | 24 ++--- .../TimePredicateFilterOptimizerTest.java | 101 +++++++++++------- 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 2fe3771542e..6ebf1ce4aec 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -436,32 +436,32 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case EQUALS: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) .getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands .get(4).getLiteral().getStringValue()); @@ -471,19 +471,19 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { lowerInclusive = false; if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue()); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) .getLiteral().getStringValue()); diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index e63fb8ee5fa..9fdff5bf50c 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -40,23 +40,23 @@ public void testTimeConvert() { testNoOpTimeConvert("TIMECONVERT(col, 'MILLISECONDS', 'MILLISECONDS') = 1620830760000"); // Other output format - testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'SECONDS') > 1620830760", + testTimeConvert("timeConvert(col, 'MILLISECONDS', 'SECONDS') > 1620830760", new Range(1620830761000L, true, null, false)); - testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'MINUTES') < 27015286", + testTimeConvert("timeConvert(col, 'MILLISECONDS', 'MINUTES') < 27015286", new Range(null, false, 1620917160000L, false)); - testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'HOURS') BETWEEN 450230 AND 450254", + testTimeConvert("timeConvert(col, 'MILLISECONDS', 'HOURS') BETWEEN 450230 AND 450254", new Range(1620828000000L, true, 1620918000000L, false)); - testPredicateFilterOptimizer("timeConvert(col, 'MILLISECONDS', 'DAYS') = 18759", + testTimeConvert("timeConvert(col, 'MILLISECONDS', 'DAYS') = 18759", new Range(1620777600000L, true, 1620864000000L, false)); // Other input format - testPredicateFilterOptimizer("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", + testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", new Range(27013846L, false, null, false)); - testPredicateFilterOptimizer("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", + testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", new Range(null, false, 450254L, true)); - testPredicateFilterOptimizer("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254", + testTimeConvert("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254", new Range(18759L, false, 18760L, true)); - testPredicateFilterOptimizer("timeConvert(col, 'SECONDS', 'DAYS') = 18759", + testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759", new Range(1620777600L, true, 1620864000L, false)); // Invalid time @@ -67,70 +67,70 @@ public void testTimeConvert() { @Test public void testEpochToEpochDateTimeConvert() { // Value not on granularity boundary - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620830760000", new Range(1620831600000L, true, null, false)); - testPredicateFilterOptimizer( + testTimeConvert( "DATE_TIME_CONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false, 1620918000000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "datetimeconvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760000 AND" + " 1620917160000", new Range(1620831600000L, true, 1620918000000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "DATETIMECONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620830760000", new Range(1620831600000L, true, 1620831600000L, false)); // Value on granularity boundary - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620831600000", new Range(1620833400000L, true, null, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620918000000", new Range(null, false, 1620918000000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620831600000 AND" + " 1620918000000", new Range(1620831600000L, true, 1620919800000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620831600000", new Range(1620831600000L, true, 1620833400000L, false)); // Other output format - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')" + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')" + " > 27013846", new Range(1620831600000L, true, null, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') " + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') " + "< 2701528", new Range(null, false, 1620918000000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND " + "1620917160", new Range(1620831600000L, true, 1620918000000L, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') " + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') " + "> 900462", new Range(1620833400000L, true, null, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", new Range(null, false, 1620918000000L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760", new Range(1620777600000L, true, 1620950400000L, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", new Range(1620777600000L, true, 1620864000000L, false)); // Other input format - testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", + testTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", new Range(1620831600L, true, null, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MINUTES:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", + testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", new Range(null, false, 27015300L, false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:DAYS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND 1620917160", new Range(18759L, false, 18760L, true)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", + testTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", new Range(1620833400L, true, null, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", + testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", new Range(null, false, 27015300L, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') " + testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') " + "BETWEEN 18759 AND 18760", new Range(18759L, true, 18761L, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", + testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", new Range(18759L, true, 18760L, false)); // Invalid time @@ -140,17 +140,17 @@ public void testEpochToEpochDateTimeConvert() { @Test public void testSDFToEpochDateTimeConvert() { - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', " + "'30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false)); - testPredicateFilterOptimizer("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', " + testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', " + "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false, "2021-05-13 15:00:00", false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', '30:MINUTES') " + "BETWEEN 1620830760000 AND 1620917160000", new Range("2021-05-12 15:00", true, "2021-05-13 15:00", false)); - testPredicateFilterOptimizer( + testTimeConvert( "dateTimeConvert(col, '1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd', '1:MILLISECONDS:EPOCH', '30:MINUTES') = " + "1620830760000", new Range("2021-05-12", false, "2021-05-12", true)); @@ -163,6 +163,35 @@ public void testSDFToEpochDateTimeConvert() { + "'30:MINUTES') < 1620917160"); } + + @Test + public void testDateTruncOptimizer() { + testDateTrunc( + "datetrunc('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + testDateTrunc( + "dateTrunc('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + testDateTrunc( + "DATETRUNC('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + testDateTrunc( + "DATE_TRUNC('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + } + + /** + * Helper method to test optimizing DATE_TRUNC on the given filter. + */ + private void testDateTrunc(String filterString, Range expectedRange) { + Expression originalExpression = CalciteSqlParser.compileToExpression(filterString); + Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString)); + Function function = optimizedFilterExpression.getFunctionCall(); + assertEquals(function.getOperator(), FilterKind.RANGE.name()); + List operands = function.getOperands(); + assertEquals(operands.size(), 2); + assertEquals(operands.get(0), + originalExpression.getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(1)); + String rangeString = operands.get(1).getLiteral().getStringValue(); + assertEquals(rangeString, expectedRange.getRangeString()); + } + /** * Helper method to test no-op TIME_CONVERT filter (same input and output time unit). */ @@ -184,9 +213,9 @@ private void testNoOpTimeConvert(String filterString) { } /** - * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT/DATE_TRUNC on the given filter. + * Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT on the given filter. */ - private void testPredicateFilterOptimizer(String filterString, Range expectedRange) { + private void testTimeConvert(String filterString, Range expectedRange) { Expression originalExpression = CalciteSqlParser.compileToExpression(filterString); Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString)); Function function = optimizedFilterExpression.getFunctionCall(); From 0b41d250cf36a9a99b4ae1ded1073923e0752bb4 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 14:00:33 -0800 Subject: [PATCH 08/21] add tests for range pred --- .../filter/TimePredicateFilterOptimizer.java | 52 ++++++++++--------- .../TimePredicateFilterOptimizerTest.java | 23 ++++++-- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 6ebf1ce4aec..59877230775 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -436,7 +436,8 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case EQUALS: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), + TimeUnit.MILLISECONDS.name()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { @@ -492,19 +493,19 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case GREATER_THAN_OR_EQUAL: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue()); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) .getLiteral().getStringValue()); @@ -514,18 +515,18 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { upperInclusive = false; if (dateTruncOperands.size() == 2) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 4) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) .getLiteral().getStringValue()); @@ -534,18 +535,18 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case LESS_THAN_OR_EQUAL: if (dateTruncOperands.size() == 2) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 4) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) .getLiteral().getStringValue()); @@ -554,33 +555,33 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case BETWEEN: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue())); + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue()); } else if (dateTruncOperands.size() == 4) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) .getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() .getStringValue()); } else if (dateTruncOperands.size() == 5) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4).getLiteral().getStringValue()); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getStringValue()), + inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands .get(4).getLiteral().getStringValue()); @@ -590,7 +591,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { throw new IllegalStateException(); } lowerMillis = (lowerMillis == null) ? 0 : lowerMillis; - upperMillis = (upperMillis == null) ? 0 : upperMillis; + upperMillis = (upperMillis == null) ? Long.MAX_VALUE : upperMillis; String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); @@ -669,8 +670,9 @@ private static long dateTruncCeil(String unit, long timeValue, String inputTimeU private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, String outputTimeUnit) { - return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) - .roundCeiling(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))), - TimeUnit.MILLISECONDS); + long lowerLimit = dateTruncFloor(unit, timeValue, inputTimeUnit, chronology, outputTimeUnit); + lowerLimit += TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(unit.toUpperCase() + "S")); + lowerLimit -= 1; + return lowerLimit; } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index 9fdff5bf50c..076021b324e 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -167,13 +167,28 @@ public void testSDFToEpochDateTimeConvert() { @Test public void testDateTruncOptimizer() { testDateTrunc( - "datetrunc('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + "datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); testDateTrunc( - "dateTrunc('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + "dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); testDateTrunc( - "DATETRUNC('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + "DATETRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); testDateTrunc( - "DATE_TRUNC('MILLISECOND', col) = 1620830760000", new Range("1620830760000", true, "1620830760000", true)); + "DATE_TRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); + + testDateTrunc( + "datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); + + testDateTrunc( + "datetrunc('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + testDateTrunc( + "dateTrunc('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + testDateTrunc( + "DATETRUNC('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + testDateTrunc( + "DATE_TRUNC('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + + testDateTrunc( + "datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); } /** From 9391b483906cb8d5a5c601ea3d18c13000d4aac9 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 14:03:25 -0800 Subject: [PATCH 09/21] style --- .../query/optimizer/filter/TimePredicateFilterOptimizer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 59877230775..f1a61c792c7 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -436,8 +436,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case EQUALS: if (dateTruncOperands.size() == 2) { lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), - TimeUnit.MILLISECONDS.name()); + inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); } else if (dateTruncOperands.size() == 3) { From 36e7639bb3235f39210cdd95db319fd3270e3587 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 15:55:56 -0800 Subject: [PATCH 10/21] fix algo and tests --- .../filter/TimePredicateFilterOptimizer.java | 217 ++++-------------- .../TimePredicateFilterOptimizerTest.java | 21 +- 2 files changed, 56 insertions(+), 182 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index f1a61c792c7..88a66181f7b 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -431,160 +431,55 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); boolean lowerInclusive = true; boolean upperInclusive = true; - + List operands; switch (filterKind) { case EQUALS: - if (dateTruncOperands.size() == 2) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 4) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) - .getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils - .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), - dateTruncOperands.get(4).getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands - .get(4).getLiteral().getStringValue()); + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + lowerMillis = dateTruncFloor(operands); + upperMillis = dateTruncCeil(operands); + // Check if it is impossible to obtain literal equality + if (lowerMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + lowerMillis = Long.MAX_VALUE; + upperMillis = Long.MIN_VALUE; } break; case GREATER_THAN: lowerInclusive = false; - if (dateTruncOperands.size() == 2) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue()); - } else if (dateTruncOperands.size() == 4) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) - .getLiteral().getStringValue()); - } + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + lowerMillis = dateTruncCeil(operands); break; case GREATER_THAN_OR_EQUAL: - if (dateTruncOperands.size() == 2) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue()); - } else if (dateTruncOperands.size() == 4) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) - .getLiteral().getStringValue()); + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + lowerMillis = dateTruncCeil(operands); + if (dateTruncFloor(operands) + == inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + lowerMillis = dateTruncFloor(operands); } break; case LESS_THAN: upperInclusive = false; - if (dateTruncOperands.size() == 2) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 4) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) - .getLiteral().getStringValue()); + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + upperMillis = dateTruncFloor(operands); + if (upperMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + upperInclusive = true; + upperMillis = dateTruncCeil(operands); } break; case LESS_THAN_OR_EQUAL: - if (dateTruncOperands.size() == 2) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 4) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands.get(4) - .getLiteral().getStringValue()); - } + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + upperMillis = dateTruncCeil(operands); break; case BETWEEN: - if (dateTruncOperands.size() == 2) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue())); - } else if (dateTruncOperands.size() == 3) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue()); - } else if (dateTruncOperands.size() == 4) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3) - .getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), dateTruncOperands.get(3).getLiteral() - .getStringValue()); - } else if (dateTruncOperands.size() == 5) { - lowerMillis = dateTruncFloor(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils - .getChronology(TimeZoneKey.getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), - dateTruncOperands.get(4).getLiteral().getStringValue()); - upperMillis = dateTruncCeil(dateTruncOperands.get(0).getLiteral().getStringValue(), - inputFormat.fromFormatToMillis(filterOperands.get(2).getLiteral().getLongValue()), - dateTruncOperands.get(2).getLiteral().getStringValue(), DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(dateTruncOperands.get(3).getLiteral().getStringValue())), dateTruncOperands - .get(4).getLiteral().getStringValue()); - } + operands = new ArrayList<>(dateTruncOperands); + operands.set(1, filterOperands.get(1)); + lowerMillis = dateTruncCeil(operands); + operands.set(1, filterOperands.get(2)); + upperMillis = dateTruncCeil(operands); break; default: throw new IllegalStateException(); @@ -631,47 +526,21 @@ private static void rewriteToRange(Function filterFunction, Expression expressio newOperands.add(RequestUtils.getLiteralExpression(rangeString)); filterFunction.setOperands(newOperands); } - - private static long dateTruncFloor(String unit, long timeValue) { - return dateTruncFloor(unit, timeValue, TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), - TimeUnit.MILLISECONDS.name()); - } - - private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit) { - return dateTruncFloor(unit, timeValue, inputTimeUnit, ISOChronology.getInstanceUTC(), inputTimeUnit); - } - - private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit, String timeZone) { - return dateTruncFloor(unit, timeValue, inputTimeUnit, - DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(timeZone)), inputTimeUnit); - } - - private static long dateTruncFloor(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, - String outputTimeUnit) { + private static long dateTruncFloor(List operands) { + String unit = operands.get(0).getLiteral().getStringValue(); + long timeValue = operands.get(1).getLiteral().getLongValue(); + String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); + ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); + String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : inputTimeUnit; return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) .roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))), TimeUnit.MILLISECONDS); } - private static long dateTruncCeil(String unit, long timeValue) { - return dateTruncCeil(unit, timeValue, TimeUnit.MILLISECONDS.name(), ISOChronology.getInstanceUTC(), - TimeUnit.MILLISECONDS.name()); - } - - private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit) { - return dateTruncCeil(unit, timeValue, inputTimeUnit, ISOChronology.getInstanceUTC(), inputTimeUnit); - } - - private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, String timeZone) { - return dateTruncCeil(unit, timeValue, inputTimeUnit, - DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(timeZone)), inputTimeUnit); - } - - private static long dateTruncCeil(String unit, long timeValue, String inputTimeUnit, ISOChronology chronology, - String outputTimeUnit) { - long lowerLimit = dateTruncFloor(unit, timeValue, inputTimeUnit, chronology, outputTimeUnit); - lowerLimit += TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(unit.toUpperCase() + "S")); - lowerLimit -= 1; - return lowerLimit; + private static long dateTruncCeil(List operands) { + String unit = operands.get(0).getLiteral().getStringValue(); + return dateTruncFloor(operands) + TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(unit.toUpperCase() + 'S')) - 1; } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index 076021b324e..bab18506df0 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -167,28 +167,33 @@ public void testSDFToEpochDateTimeConvert() { @Test public void testDateTruncOptimizer() { testDateTrunc( - "datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); + "datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); testDateTrunc( - "dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); + "dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); testDateTrunc( - "DATETRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); + "DATETRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); testDateTrunc( - "DATE_TRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620863999999", false)); + "DATE_TRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); testDateTrunc( "datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); testDateTrunc( - "datetrunc('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + "datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( - "dateTrunc('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + "dateTrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( - "DATETRUNC('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + "DATETRUNC('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( - "DATE_TRUNC('DAY', col) > 1620777600000", new Range("1620777600000", false, Long.MAX_VALUE, true)); + "DATE_TRUNC('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( "datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); + + testDateTrunc( + "datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); + testDateTrunc( + "dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); } /** From bc08f038bccc38dcde2b0007dcb2a09754e34110 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 16:32:50 -0800 Subject: [PATCH 11/21] fix between operator --- .../filter/TimePredicateFilterOptimizer.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 88a66181f7b..6542909bf1a 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -428,13 +428,14 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { Long lowerMillis = null; Long upperMillis = null; - DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); + DateTimeFormatSpec inputFormat = (dateTruncOperands.size() >= 3) + ? new DateTimeFormatSpec(dateTruncOperands.get(2).getLiteral().getStringValue()) + : new DateTimeFormatSpec("TIMESTAMP"); boolean lowerInclusive = true; boolean upperInclusive = true; - List operands; + List operands = new ArrayList<>(dateTruncOperands); switch (filterKind) { case EQUALS: - operands = new ArrayList<>(dateTruncOperands); operands.set(1, filterOperands.get(1)); lowerMillis = dateTruncFloor(operands); upperMillis = dateTruncCeil(operands); @@ -446,12 +447,10 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { break; case GREATER_THAN: lowerInclusive = false; - operands = new ArrayList<>(dateTruncOperands); operands.set(1, filterOperands.get(1)); lowerMillis = dateTruncCeil(operands); break; case GREATER_THAN_OR_EQUAL: - operands = new ArrayList<>(dateTruncOperands); operands.set(1, filterOperands.get(1)); lowerMillis = dateTruncCeil(operands); if (dateTruncFloor(operands) @@ -461,7 +460,6 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { break; case LESS_THAN: upperInclusive = false; - operands = new ArrayList<>(dateTruncOperands); operands.set(1, filterOperands.get(1)); upperMillis = dateTruncFloor(operands); if (upperMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { @@ -470,15 +468,31 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { } break; case LESS_THAN_OR_EQUAL: - operands = new ArrayList<>(dateTruncOperands); operands.set(1, filterOperands.get(1)); upperMillis = dateTruncCeil(operands); break; case BETWEEN: - operands = new ArrayList<>(dateTruncOperands); - operands.set(1, filterOperands.get(1)); + Literal lowerLiteral = new Literal(); + if (filterOperands.get(1).getLiteral().isSetStringValue()) { + lowerLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis( + filterOperands.get(1).getLiteral().getStringValue())); + } else { + lowerLiteral.setLongValue(getLongValue(filterOperands.get(1))); + } + Expression lowerExpression = new Expression(ExpressionType.LITERAL); + lowerExpression.setLiteral(lowerLiteral); + operands.set(1, lowerExpression); lowerMillis = dateTruncCeil(operands); - operands.set(1, filterOperands.get(2)); + Literal upperLiteral = new Literal(); + if (filterOperands.get(2).getLiteral().isSetStringValue()) { + upperLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis( + filterOperands.get(2).getLiteral().getStringValue())); + } else { + upperLiteral.setLongValue(getLongValue(filterOperands.get(2))); + } + Expression upperExpression = new Expression(ExpressionType.LITERAL); + upperExpression.setLiteral(upperLiteral); + operands.set(1, upperExpression); upperMillis = dateTruncCeil(operands); break; default: From b09fc491c7c2759ec3bd0b37021d4fc467fbda8f Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 16:58:44 -0800 Subject: [PATCH 12/21] add tests --- .../filter/TimePredicateFilterOptimizer.java | 2 ++ .../filter/TimePredicateFilterOptimizerTest.java | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 6542909bf1a..580514ab836 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -452,9 +452,11 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { break; case GREATER_THAN_OR_EQUAL: operands.set(1, filterOperands.get(1)); + lowerInclusive = false; lowerMillis = dateTruncCeil(operands); if (dateTruncFloor(operands) == inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + lowerInclusive = true; lowerMillis = dateTruncFloor(operands); } break; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index bab18506df0..cc25f0ed866 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -171,22 +171,24 @@ public void testDateTruncOptimizer() { testDateTrunc( "dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); testDateTrunc( - "DATETRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); + "DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true)); testDateTrunc( - "DATE_TRUNC('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); + "DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true)); testDateTrunc( "datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); + testDateTrunc( + "datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true)); testDateTrunc( "datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( - "dateTrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); - testDateTrunc( - "DATETRUNC('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); + "dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( - "DATE_TRUNC('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); + "DATETRUNC('DAY', col) > 1620864000000", new Range("1620950399999", false, Long.MAX_VALUE, true)); + testDateTrunc( + "datetrunc('DAY', col) >= 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc( "datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); From eec0e7fe51363f0126ffd8a836e85e3b817a7a90 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 22:01:59 -0800 Subject: [PATCH 13/21] use correct output format --- .../optimizer/filter/TimePredicateFilterOptimizer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 580514ab836..4af44d69487 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -431,6 +431,9 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { DateTimeFormatSpec inputFormat = (dateTruncOperands.size() >= 3) ? new DateTimeFormatSpec(dateTruncOperands.get(2).getLiteral().getStringValue()) : new DateTimeFormatSpec("TIMESTAMP"); + DateTimeFormatSpec outputFormat = (dateTruncOperands.size() == 5) + ? new DateTimeFormatSpec(dateTruncOperands.get(4).getLiteral().getStringValue()) + : new DateTimeFormatSpec("TIMESTAMP"); boolean lowerInclusive = true; boolean upperInclusive = true; List operands = new ArrayList<>(dateTruncOperands); @@ -476,7 +479,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { case BETWEEN: Literal lowerLiteral = new Literal(); if (filterOperands.get(1).getLiteral().isSetStringValue()) { - lowerLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis( + lowerLiteral.setLongValue(outputFormat.fromFormatToMillis( filterOperands.get(1).getLiteral().getStringValue())); } else { lowerLiteral.setLongValue(getLongValue(filterOperands.get(1))); @@ -487,7 +490,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { lowerMillis = dateTruncCeil(operands); Literal upperLiteral = new Literal(); if (filterOperands.get(2).getLiteral().isSetStringValue()) { - upperLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis( + upperLiteral.setLongValue(outputFormat.fromFormatToMillis( filterOperands.get(2).getLiteral().getStringValue())); } else { upperLiteral.setLongValue(getLongValue(filterOperands.get(2))); From 614e176ef0d7694c2b7a667411f993d6e20b5321 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Tue, 5 Nov 2024 22:36:37 -0800 Subject: [PATCH 14/21] add helper function --- .../optimizer/filter/TimePredicateFilterOptimizer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 4af44d69487..531ad9d2d6e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -560,6 +560,10 @@ private static long dateTruncFloor(List operands) { private static long dateTruncCeil(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); - return dateTruncFloor(operands) + TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(unit.toUpperCase() + 'S')) - 1; + return dateTruncFloor(operands) + TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(getGranularityUnit(unit))) - 1; + } + + private static String getGranularityUnit(String unit) { + return unit.toUpperCase() + 'S'; } } From 624e360b5555f7a21ec80349a830e51ae87ceb74 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Wed, 6 Nov 2024 09:52:43 -0800 Subject: [PATCH 15/21] make unit step less brittle --- .../optimizer/filter/TimePredicateFilterOptimizer.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 531ad9d2d6e..e8af07747fd 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -560,10 +560,8 @@ private static long dateTruncFloor(List operands) { private static long dateTruncCeil(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); - return dateTruncFloor(operands) + TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(getGranularityUnit(unit))) - 1; - } - - private static String getGranularityUnit(String unit) { - return unit.toUpperCase() + 'S'; + ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey + .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); + return dateTruncFloor(operands) + DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(1) - 1; } } From 5e87ed70690bf413cebf61e84020aca0c0cf6bbb Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Wed, 6 Nov 2024 11:36:42 -0800 Subject: [PATCH 16/21] correct conversion to output unit --- .../filter/TimePredicateFilterOptimizer.java | 85 ++++++++++--------- .../TimePredicateFilterOptimizerTest.java | 3 + 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index e8af07747fd..bda4fce609c 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -428,46 +428,46 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { Long lowerMillis = null; Long upperMillis = null; - DateTimeFormatSpec inputFormat = (dateTruncOperands.size() >= 3) - ? new DateTimeFormatSpec(dateTruncOperands.get(2).getLiteral().getStringValue()) - : new DateTimeFormatSpec("TIMESTAMP"); - DateTimeFormatSpec outputFormat = (dateTruncOperands.size() == 5) - ? new DateTimeFormatSpec(dateTruncOperands.get(4).getLiteral().getStringValue()) - : new DateTimeFormatSpec("TIMESTAMP"); + DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); + String inputTimeUnit = (dateTruncOperands.size() >= 3) ? dateTruncOperands.get(2).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); + String outputTimeUnit = (dateTruncOperands.size() == 5) ? dateTruncOperands.get(4).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); boolean lowerInclusive = true; boolean upperInclusive = true; List operands = new ArrayList<>(dateTruncOperands); switch (filterKind) { case EQUALS: - operands.set(1, filterOperands.get(1)); + operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerMillis = dateTruncFloor(operands); upperMillis = dateTruncCeil(operands); // Check if it is impossible to obtain literal equality - if (lowerMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + if (lowerMillis != TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(filterOperands.get(1)), + TimeUnit.valueOf(outputTimeUnit))) { lowerMillis = Long.MAX_VALUE; upperMillis = Long.MIN_VALUE; } break; case GREATER_THAN: lowerInclusive = false; - operands.set(1, filterOperands.get(1)); + operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerMillis = dateTruncCeil(operands); break; case GREATER_THAN_OR_EQUAL: - operands.set(1, filterOperands.get(1)); + operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerInclusive = false; lowerMillis = dateTruncCeil(operands); if (dateTruncFloor(operands) - == inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + == inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) { lowerInclusive = true; lowerMillis = dateTruncFloor(operands); } break; case LESS_THAN: upperInclusive = false; - operands.set(1, filterOperands.get(1)); + operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncFloor(operands); - if (upperMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) { + if (upperMillis != inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) { upperInclusive = true; upperMillis = dateTruncCeil(operands); } @@ -477,27 +477,8 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { upperMillis = dateTruncCeil(operands); break; case BETWEEN: - Literal lowerLiteral = new Literal(); - if (filterOperands.get(1).getLiteral().isSetStringValue()) { - lowerLiteral.setLongValue(outputFormat.fromFormatToMillis( - filterOperands.get(1).getLiteral().getStringValue())); - } else { - lowerLiteral.setLongValue(getLongValue(filterOperands.get(1))); - } - Expression lowerExpression = new Expression(ExpressionType.LITERAL); - lowerExpression.setLiteral(lowerLiteral); - operands.set(1, lowerExpression); - lowerMillis = dateTruncCeil(operands); - Literal upperLiteral = new Literal(); - if (filterOperands.get(2).getLiteral().isSetStringValue()) { - upperLiteral.setLongValue(outputFormat.fromFormatToMillis( - filterOperands.get(2).getLiteral().getStringValue())); - } else { - upperLiteral.setLongValue(getLongValue(filterOperands.get(2))); - } - Expression upperExpression = new Expression(ExpressionType.LITERAL); - upperExpression.setLiteral(upperLiteral); - operands.set(1, upperExpression); + operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getExpression(filterOperands.get(2), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncCeil(operands); break; default: @@ -545,9 +526,29 @@ private static void rewriteToRange(Function filterFunction, Expression expressio newOperands.add(RequestUtils.getLiteralExpression(rangeString)); filterFunction.setOperands(newOperands); } - private static long dateTruncFloor(List operands) { + + + private Expression getExpression(Expression value, DateTimeFormatSpec inputFormat, + String inputTimeUnit, String outputTimeUnit) { + Literal literal = new Literal(); + if (value.getLiteral().isSetStringValue()) { + literal.setLongValue(inputFormat.fromFormatToMillis( + value.getLiteral().getStringValue())); + } else { + literal.setLongValue(TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(value), + TimeUnit.valueOf(outputTimeUnit))); + } + Expression expression = new Expression(ExpressionType.LITERAL); + expression.setLiteral(literal); + return expression; + } + + /** + * Helper function mimicking date trunc function + */ + private long dateTruncFloor(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); - long timeValue = operands.get(1).getLiteral().getLongValue(); + long timeValue = getLongValue(operands.get(1)); String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() : TimeUnit.MILLISECONDS.name(); ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey @@ -558,10 +559,18 @@ private static long dateTruncFloor(List operands) { TimeUnit.MILLISECONDS); } - private static long dateTruncCeil(List operands) { + /** + * Helper function that finds the maximum value (ceiling) that truncates to specified value + * Computes ceiling inverse of date trunc function + */ + private long dateTruncCeil(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); + String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); - return dateTruncFloor(operands) + DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(1) - 1; + // Add value of 1 unit as specified, subtract one to find maximum value that will truncate to desired value + return dateTruncFloor(operands) + TimeUnit.valueOf(inputTimeUnit).convert( + DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(1), TimeUnit.MILLISECONDS) - 1; } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index cc25f0ed866..fdf77231ac3 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -196,6 +196,9 @@ public void testDateTruncOptimizer() { "datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); testDateTrunc( "dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); + + testDateTrunc( + "datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); } /** From 4b836c7a89e4a15b27a0f532be4c69689717c22d Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Wed, 6 Nov 2024 13:37:47 -0800 Subject: [PATCH 17/21] add javadocs --- .../filter/TimePredicateFilterOptimizer.java | 11 ++ .../TimePredicateFilterOptimizerTest.java | 133 +++++++----------- 2 files changed, 61 insertions(+), 83 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index bda4fce609c..ac7eef6080d 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -57,6 +57,17 @@ *

NOTE: Other predicates such as NOT_EQUALS, IN, NOT_IN are not supported for now because these predicates are * not common on time column, and they cannot be optimized to a single range predicate. * + *

  • + * Optimizes DATE_TRUNC function with range/equality predicates by either rounding up or down to closest granularity + * step + *

    E.g. "dateTrunc('DAY', col, 'MILLISECONDS') > 1620777600000" will be optimized + * to "col > 1620863999999" as 1620863999999 is the largest value that can be truncated to 1620777600000 + *

    E.g. "datetrunc('DAY', col, 'MILLISECONDS') <= 1620777600010" will be optimized + * to col <= 1620863999999 as the next granularity step lower than 1620777600010 is 1620777600000 and 1620863999999 + * is the largest value that truncates to be lower than the specified literal. + *

    NOTE: Other predicates such as NOT_EQUALS, IN, NOT_IN are not supported for now because these predicates are + * not common on time column, and they cannot be optimized to a single range predicate. + *

    • * * * NOTE: This optimizer is followed by the {@link MergeRangeFilterOptimizer}, which can merge the generated ranges. diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index fdf77231ac3..e7f08ac13d4 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -67,51 +67,38 @@ public void testTimeConvert() { @Test public void testEpochToEpochDateTimeConvert() { // Value not on granularity boundary - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620830760000", - new Range(1620831600000L, true, null, false)); - testTimeConvert( - "DATE_TIME_CONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", - new Range(null, false, 1620918000000L, false)); - testTimeConvert( - "datetimeconvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760000 AND" - + " 1620917160000", new Range(1620831600000L, true, 1620918000000L, false)); - testTimeConvert( - "DATETIMECONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620830760000", - new Range(1620831600000L, true, 1620831600000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > " + + "1620830760000", new Range(1620831600000L, true, null, false)); + testTimeConvert("DATE_TIME_CONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < " + + "1620917160000", new Range(null, false, 1620918000000L, false)); + testTimeConvert("datetimeconvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN " + + "1620830760000 AND 1620917160000", new Range(1620831600000L, true, 1620918000000L, false)); + testTimeConvert("DATETIMECONVERT(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = " + + "1620830760000", new Range(1620831600000L, true, 1620831600000L, false)); // Value on granularity boundary - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620831600000", - new Range(1620833400000L, true, null, false)); - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620918000000", - new Range(null, false, 1620918000000L, false)); - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') BETWEEN 1620831600000 AND" - + " 1620918000000", new Range(1620831600000L, true, 1620919800000L, false)); - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = 1620831600000", - new Range(1620831600000L, true, 1620833400000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') > " + + "1620831600000", new Range(1620833400000L, true, null, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') < " + + "1620918000000", new Range(null, false, 1620918000000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') " + + "BETWEEN 1620831600000 AND 1620918000000", new Range(1620831600000L, true, 1620919800000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '30:MINUTES') = " + + "1620831600000", new Range(1620831600000L, true, 1620833400000L, false)); // Other output format - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')" - + " > 27013846", + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846", new Range(1620831600000L, true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') " - + "< 2701528", + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528", new Range(null, false, 1620918000000L, false)); - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND " - + "1620917160", new Range(1620831600000L, true, 1620918000000L, false)); - testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') " - + "> 900462", + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 " + + "AND 1620917160", new Range(1620831600000L, true, 1620918000000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462", new Range(1620833400000L, true, null, false)); testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255", new Range(null, false, 1620918000000L, false)); - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760", - new Range(1620777600000L, true, 1620950400000L, false)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND " + + "18760", new Range(1620777600000L, true, 1620950400000L, false)); testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759", new Range(1620777600000L, true, 1620864000000L, false)); @@ -140,65 +127,45 @@ public void testEpochToEpochDateTimeConvert() { @Test public void testSDFToEpochDateTimeConvert() { - testTimeConvert( - "dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', " - + "'30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false)); - testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', " - + "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false, - "2021-05-13 15:00:00", false)); - testTimeConvert( - "dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', '30:MINUTES') " - + "BETWEEN 1620830760000 AND 1620917160000", - new Range("2021-05-12 15:00", true, "2021-05-13 15:00", false)); - testTimeConvert( - "dateTimeConvert(col, '1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd', '1:MILLISECONDS:EPOCH', '30:MINUTES') = " - + "1620830760000", new Range("2021-05-12", false, "2021-05-12", true)); + testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:" + + "EPOCH', '30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false)); + testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH'," + + " '30:MINUTES') < 1620917160000", new Range(null, false, "2021-05-13 15:00:00", false)); + testTimeConvert("dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', " + + "'30:MINUTES') BETWEEN 1620830760000 AND 1620917160000", new Range("2021-05-12 15:00", true, "2021-05-13 " + + "15:00", false)); + testTimeConvert("dateTimeConvert(col, '1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd', '1:MILLISECONDS:EPOCH', '30:MINUTES')" + + " = 1620830760000", new Range("2021-05-12", false, "2021-05-12", true)); // Invalid time - testInvalidFilterOptimizer( - "dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', " - + "'30:MINUTES') > 1620830760000.5"); - testInvalidFilterOptimizer( - "dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', " - + "'30:MINUTES') < 1620917160"); + testInvalidFilterOptimizer("dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', " + + "'1:MILLISECONDS:EPOCH', '30:MINUTES') > 1620830760000.5"); + testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', " + + "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160"); } @Test public void testDateTruncOptimizer() { - testDateTrunc( - "datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); - testDateTrunc( - "dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); - testDateTrunc( - "DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true)); - testDateTrunc( - "DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); + testDateTrunc("dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); + testDateTrunc("DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true)); + testDateTrunc("DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true)); - testDateTrunc( - "datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); - testDateTrunc( - "datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true)); - testDateTrunc( - "datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); - testDateTrunc( - "dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); - testDateTrunc( - "DATETRUNC('DAY', col) > 1620864000000", new Range("1620950399999", false, Long.MAX_VALUE, true)); + testDateTrunc("datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); + testDateTrunc("dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); + testDateTrunc("DATETRUNC('DAY', col) > 1620864000000", new Range("1620950399999", false, Long.MAX_VALUE, true)); - testDateTrunc( - "datetrunc('DAY', col) >= 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); - testDateTrunc( - "datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); + testDateTrunc("datetrunc('DAY', col) >= 1620863999909", new Range("1620863999999", false, Long.MAX_VALUE, true)); + testDateTrunc("datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); - testDateTrunc( - "datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); - testDateTrunc( - "dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); + testDateTrunc("datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); + testDateTrunc("dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); - testDateTrunc( - "datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); + testDateTrunc("datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); } /** From 470d4bb77dead16104836ad46a769ebc283f69f1 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Thu, 7 Nov 2024 12:46:14 -0800 Subject: [PATCH 18/21] fix short circuit --- .../filter/TimePredicateFilterOptimizer.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index ac7eef6080d..ddc8c35aa92 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -449,23 +449,22 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { List operands = new ArrayList<>(dateTruncOperands); switch (filterKind) { case EQUALS: - operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerMillis = dateTruncFloor(operands); upperMillis = dateTruncCeil(operands); // Check if it is impossible to obtain literal equality - if (lowerMillis != TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(filterOperands.get(1)), - TimeUnit.valueOf(outputTimeUnit))) { + if (lowerMillis != getLongValue(filterOperands.get(1))) { lowerMillis = Long.MAX_VALUE; upperMillis = Long.MIN_VALUE; } break; case GREATER_THAN: lowerInclusive = false; - operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerMillis = dateTruncCeil(operands); break; case GREATER_THAN_OR_EQUAL: - operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); lowerInclusive = false; lowerMillis = dateTruncCeil(operands); if (dateTruncFloor(operands) @@ -476,7 +475,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { break; case LESS_THAN: upperInclusive = false; - operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncFloor(operands); if (upperMillis != inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) { upperInclusive = true; @@ -488,8 +487,12 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { upperMillis = dateTruncCeil(operands); break; case BETWEEN: - operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); - operands.set(1, getExpression(filterOperands.get(2), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + lowerMillis = dateTruncFloor(operands); + if (lowerMillis != getLongValue(filterOperands.get(1))) { + lowerMillis = dateTruncCeil(operands); + } + operands.set(1, getInverseExpression(filterOperands.get(2), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncCeil(operands); break; default: @@ -539,7 +542,7 @@ private static void rewriteToRange(Function filterFunction, Expression expressio } - private Expression getExpression(Expression value, DateTimeFormatSpec inputFormat, + private Expression getInverseExpression(Expression value, DateTimeFormatSpec inputFormat, String inputTimeUnit, String outputTimeUnit) { Literal literal = new Literal(); if (value.getLiteral().isSetStringValue()) { From 321cc6a91cbb3c7860afc2297380ac99e509b616 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Thu, 7 Nov 2024 14:16:02 -0800 Subject: [PATCH 19/21] fix range settings --- .../filter/TimePredicateFilterOptimizer.java | 21 ++++++++++--------- .../TimePredicateFilterOptimizerTest.java | 15 +++++++------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index ddc8c35aa92..7354816714a 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -453,7 +453,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { lowerMillis = dateTruncFloor(operands); upperMillis = dateTruncCeil(operands); // Check if it is impossible to obtain literal equality - if (lowerMillis != getLongValue(filterOperands.get(1))) { + if (lowerMillis != TimeUnit.MILLISECONDS.convert(getLongValue(filterOperands.get(1)), TimeUnit.valueOf(outputTimeUnit.toUpperCase()))) { lowerMillis = Long.MAX_VALUE; upperMillis = Long.MIN_VALUE; } @@ -477,7 +477,7 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { upperInclusive = false; operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncFloor(operands); - if (upperMillis != inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) { + if (upperMillis != operands.get(1).getLiteral().getLongValue()) { upperInclusive = true; upperMillis = dateTruncCeil(operands); } @@ -498,8 +498,10 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { default: throw new IllegalStateException(); } - lowerMillis = (lowerMillis == null) ? 0 : lowerMillis; + lowerMillis = (lowerMillis == null) ? Long.MIN_VALUE : lowerMillis; upperMillis = (upperMillis == null) ? Long.MAX_VALUE : upperMillis; + lowerMillis = TimeUnit.valueOf(inputTimeUnit.toUpperCase()).convert(lowerMillis, TimeUnit.MILLISECONDS); + upperMillis = TimeUnit.valueOf(inputTimeUnit.toUpperCase()).convert(upperMillis, TimeUnit.MILLISECONDS); String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); @@ -549,8 +551,7 @@ private Expression getInverseExpression(Expression value, DateTimeFormatSpec inp literal.setLongValue(inputFormat.fromFormatToMillis( value.getLiteral().getStringValue())); } else { - literal.setLongValue(TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(value), - TimeUnit.valueOf(outputTimeUnit))); + literal.setLongValue(getLongValue(value)); } Expression expression = new Expression(ExpressionType.LITERAL); expression.setLiteral(literal); @@ -558,7 +559,7 @@ private Expression getInverseExpression(Expression value, DateTimeFormatSpec inp } /** - * Helper function mimicking date trunc function + * Helper function inverting date trunc function */ private long dateTruncFloor(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); @@ -567,10 +568,10 @@ private long dateTruncFloor(List operands) { : TimeUnit.MILLISECONDS.name(); ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); - String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : inputTimeUnit; - return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit) - .roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))), - TimeUnit.MILLISECONDS); + String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); + return DateTimeUtils.getTimestampField(chronology, unit) + .roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(outputTimeUnit.toUpperCase()))); } /** diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index e7f08ac13d4..1ffd3b6570f 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -147,13 +147,13 @@ public void testSDFToEpochDateTimeConvert() { @Test public void testDateTruncOptimizer() { - testDateTrunc("datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); - testDateTrunc("dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false)); - testDateTrunc("DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true)); - testDateTrunc("DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) < 1620777600000", new Range(Long.MIN_VALUE, true, "1620777600000", false)); + testDateTrunc("dateTrunc('DAY', col) < 1620777600000", new Range(Long.MIN_VALUE, true, "1620777600000", false)); + testDateTrunc("DATETRUNC('DAY', col) < 1620777600010", new Range(Long.MIN_VALUE, true, "1620863999999", true)); + testDateTrunc("DATE_TRUNC('DAY', col) < 1620863999999", new Range(Long.MIN_VALUE, true, "1620863999999", true)); - testDateTrunc("datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true)); - testDateTrunc("datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) <= 1620777600000", new Range(Long.MIN_VALUE, true, "1620863999999", true)); + testDateTrunc("datetrunc('DAY', col) <= 1620777600010", new Range(Long.MIN_VALUE, true, "1620863999999", true)); testDateTrunc("datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc("dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true)); @@ -166,6 +166,9 @@ public void testDateTruncOptimizer() { testDateTrunc("dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); testDateTrunc("datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); + testDateTrunc("datetrunc('DAY', col, 'DAYS', 'CET', 'MILLISECONDS') = 39193714800000", + new Range("453630", true, "453630", true)); + } /** From e3a665152651d9ad5126d8c06ae81702fd1c6d06 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Sun, 10 Nov 2024 21:35:13 -0800 Subject: [PATCH 20/21] everything but time zones --- .../filter/TimePredicateFilterOptimizer.java | 114 ++++++++++-------- .../TimePredicateFilterOptimizerTest.java | 17 ++- 2 files changed, 75 insertions(+), 56 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 7354816714a..89fc0b340e0 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -20,6 +20,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import java.sql.Time; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -27,6 +30,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.function.DateTimeUtils; import org.apache.pinot.common.function.TimeZoneKey; +import org.apache.pinot.common.function.scalar.DateTimeFunctions; import org.apache.pinot.common.request.Expression; import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.Function; @@ -34,6 +38,7 @@ import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.core.operator.transform.function.DateTimeConversionTransformFunction; import org.apache.pinot.core.operator.transform.function.DateTruncTransformFunction; +import org.apache.pinot.core.operator.transform.function.LiteralTransformFunction; import org.apache.pinot.core.operator.transform.function.TimeConversionTransformFunction; import org.apache.pinot.spi.data.DateTimeFieldSpec.TimeFormat; import org.apache.pinot.spi.data.DateTimeFormatSpec; @@ -41,9 +46,15 @@ import org.apache.pinot.spi.data.Schema; import org.apache.pinot.spi.utils.TimeUtils; import org.apache.pinot.sql.FilterKind; +import org.joda.time.DateTime; +import org.joda.time.DateTimeField; +import org.joda.time.DateTimeZone; +import org.joda.time.DurationField; +import org.joda.time.DurationFieldType; import org.joda.time.chrono.ISOChronology; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.*; /** @@ -432,77 +443,85 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { List filterOperands = filterFunction.getOperands(); List dateTruncOperands = filterOperands.get(0).getFunctionCall().getOperands(); - // Check if date trunc function is being applied on a literal value + // TODO: Compute value and create query is date trunc is applied on a literal value if (dateTruncOperands.get(1).isSetLiteral()) { return; } Long lowerMillis = null; Long upperMillis = null; - DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP"); - String inputTimeUnit = (dateTruncOperands.size() >= 3) ? dateTruncOperands.get(2).getLiteral().getStringValue() - : TimeUnit.MILLISECONDS.name(); - String outputTimeUnit = (dateTruncOperands.size() == 5) ? dateTruncOperands.get(4).getLiteral().getStringValue() - : TimeUnit.MILLISECONDS.name(); boolean lowerInclusive = true; boolean upperInclusive = true; List operands = new ArrayList<>(dateTruncOperands); + String unit = operands.get(0).getLiteral().getStringValue(); + String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); + ISOChronology chronology = (operands.size() >= 4) + ? DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) + : ISOChronology.getInstanceUTC(); + String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() + : TimeUnit.MILLISECONDS.name(); switch (filterKind) { case EQUALS: - operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); - lowerMillis = dateTruncFloor(operands); + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); upperMillis = dateTruncCeil(operands); - // Check if it is impossible to obtain literal equality + lowerMillis = DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(dateTruncFloor(operands)); if (lowerMillis != TimeUnit.MILLISECONDS.convert(getLongValue(filterOperands.get(1)), TimeUnit.valueOf(outputTimeUnit.toUpperCase()))) { lowerMillis = Long.MAX_VALUE; upperMillis = Long.MIN_VALUE; + String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); + rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); + return; } + System.out.println(lowerMillis + " " + upperMillis); break; case GREATER_THAN: - lowerInclusive = false; - operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); lowerMillis = dateTruncCeil(operands); + lowerInclusive = false; + upperMillis = Long.MAX_VALUE; break; case GREATER_THAN_OR_EQUAL: - operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); - lowerInclusive = false; - lowerMillis = dateTruncCeil(operands); - if (dateTruncFloor(operands) - == inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) { - lowerInclusive = true; - lowerMillis = dateTruncFloor(operands); + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); + lowerMillis = dateTruncFloor(operands); + upperMillis = Long.MAX_VALUE; + if (TimeUnit.valueOf(outputTimeUnit).convert(lowerMillis, TimeUnit.MILLISECONDS) + != getLongValue(filterOperands.get(1))) { + lowerInclusive = false; + lowerMillis = dateTruncCeil(operands); } break; case LESS_THAN: + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); + lowerMillis = Long.MIN_VALUE; upperInclusive = false; - operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); upperMillis = dateTruncFloor(operands); - if (upperMillis != operands.get(1).getLiteral().getLongValue()) { + if (upperMillis != TimeUnit.MILLISECONDS.convert(getLongValue(filterOperands.get(1)), TimeUnit.valueOf(outputTimeUnit.toUpperCase()))) { upperInclusive = true; upperMillis = dateTruncCeil(operands); } break; case LESS_THAN_OR_EQUAL: - operands.set(1, filterOperands.get(1)); + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); + lowerMillis = Long.MIN_VALUE; upperMillis = dateTruncCeil(operands); break; case BETWEEN: - operands.set(1, getInverseExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); lowerMillis = dateTruncFloor(operands); - if (lowerMillis != getLongValue(filterOperands.get(1))) { + if (TimeUnit.valueOf(outputTimeUnit).convert(lowerMillis, TimeUnit.MILLISECONDS) + != getLongValue(filterOperands.get(1))) { + lowerInclusive = false; lowerMillis = dateTruncCeil(operands); } - operands.set(1, getInverseExpression(filterOperands.get(2), inputFormat, inputTimeUnit, outputTimeUnit)); + operands.set(1, getExpression(getLongValue(filterOperands.get(2)), new DateTimeFormatSpec("TIMESTAMP"))); upperMillis = dateTruncCeil(operands); break; default: throw new IllegalStateException(); } - lowerMillis = (lowerMillis == null) ? Long.MIN_VALUE : lowerMillis; - upperMillis = (upperMillis == null) ? Long.MAX_VALUE : upperMillis; - lowerMillis = TimeUnit.valueOf(inputTimeUnit.toUpperCase()).convert(lowerMillis, TimeUnit.MILLISECONDS); - upperMillis = TimeUnit.valueOf(inputTimeUnit.toUpperCase()).convert(upperMillis, TimeUnit.MILLISECONDS); - + lowerMillis = TimeUnit.valueOf(inputTimeUnit).convert(lowerMillis, TimeUnit.MILLISECONDS); + upperMillis = TimeUnit.valueOf(inputTimeUnit).convert(upperMillis, TimeUnit.MILLISECONDS); String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString(); rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); } @@ -534,7 +553,7 @@ private long ceil(long millisValue, long granularityMillis) { return (millisValue + granularityMillis - 1) / granularityMillis * granularityMillis; } - private static void rewriteToRange(Function filterFunction, Expression expression, String rangeString) { + private void rewriteToRange(Function filterFunction, Expression expression, String rangeString) { filterFunction.setOperator(FilterKind.RANGE.name()); // NOTE: Create an ArrayList because we might need to modify the list later List newOperands = new ArrayList<>(2); @@ -544,34 +563,25 @@ private static void rewriteToRange(Function filterFunction, Expression expressio } - private Expression getInverseExpression(Expression value, DateTimeFormatSpec inputFormat, - String inputTimeUnit, String outputTimeUnit) { + private Expression getExpression(long value, DateTimeFormatSpec inputFormat) { Literal literal = new Literal(); - if (value.getLiteral().isSetStringValue()) { - literal.setLongValue(inputFormat.fromFormatToMillis( - value.getLiteral().getStringValue())); - } else { - literal.setLongValue(getLongValue(value)); - } + literal.setLongValue(value); Expression expression = new Expression(ExpressionType.LITERAL); expression.setLiteral(literal); return expression; } /** - * Helper function inverting date trunc function + * Helper function to find the floor of acceptable values truncating to a specified value */ private long dateTruncFloor(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); long timeValue = getLongValue(operands.get(1)); - String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() - : TimeUnit.MILLISECONDS.name(); - ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); + ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : TimeUnit.MILLISECONDS.name(); - return DateTimeUtils.getTimestampField(chronology, unit) - .roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(outputTimeUnit.toUpperCase()))); + long timeInMillis = TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(outputTimeUnit.toUpperCase())); + return DateTimeUtils.getTimestampField(chronology, unit).roundFloor(timeInMillis); } /** @@ -580,12 +590,16 @@ private long dateTruncFloor(List operands) { */ private long dateTruncCeil(List operands) { String unit = operands.get(0).getLiteral().getStringValue(); - String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue() + long timeValue = getLongValue(operands.get(1)); + ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); + String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : TimeUnit.MILLISECONDS.name(); - ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey - .getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC(); // Add value of 1 unit as specified, subtract one to find maximum value that will truncate to desired value - return dateTruncFloor(operands) + TimeUnit.valueOf(inputTimeUnit).convert( - DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(1), TimeUnit.MILLISECONDS) - 1; + long timeInMillis = TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(outputTimeUnit.toUpperCase())); + return DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(timeInMillis + 1) - 1; + } + + public static void main(String[] args) { + System.out.println(DateTimeFunctions.dateTrunc("DAY", new Long("1620860399999"))); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index 1ffd3b6570f..77dab5909f7 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -147,8 +147,10 @@ public void testSDFToEpochDateTimeConvert() { @Test public void testDateTruncOptimizer() { + testDateTrunc("datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); + testDateTrunc("dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); + testDateTrunc("datetrunc('DAY', col) < 1620777600000", new Range(Long.MIN_VALUE, true, "1620777600000", false)); - testDateTrunc("dateTrunc('DAY', col) < 1620777600000", new Range(Long.MIN_VALUE, true, "1620777600000", false)); testDateTrunc("DATETRUNC('DAY', col) < 1620777600010", new Range(Long.MIN_VALUE, true, "1620863999999", true)); testDateTrunc("DATE_TRUNC('DAY', col) < 1620863999999", new Range(Long.MIN_VALUE, true, "1620863999999", true)); @@ -162,13 +164,16 @@ public void testDateTruncOptimizer() { testDateTrunc("datetrunc('DAY', col) >= 1620863999909", new Range("1620863999999", false, Long.MAX_VALUE, true)); testDateTrunc("datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true)); - testDateTrunc("datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true)); - testDateTrunc("dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true)); - + testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'CET', 'MILLISECONDS') = 1620770400000", + new Range("1620770400000", true, "1620856799999", true)); testDateTrunc("datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); testDateTrunc("datetrunc('DAY', col, 'DAYS', 'CET', 'MILLISECONDS') = 39193714800000", - new Range("453630", true, "453630", true)); - + new Range("453631", true, "453631", true)); + testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'UTC', 'DAYS') = 453630", + new Range("39193632000000", true, "39193718399999", true)); + testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'CET', 'DAYS') = 453630", + new Range("39193714800000", true, "39193718399999", true)); + // TODO: add between predicate test } /** From 2e6bb782c1358c4d6b2944488c03dae798887da5 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Mon, 11 Nov 2024 16:43:55 -0800 Subject: [PATCH 21/21] just logging for future --- .../filter/TimePredicateFilterOptimizer.java | 47 +++++++++++++++++-- .../TimePredicateFilterOptimizerTest.java | 6 +-- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java index 89fc0b340e0..6f6859bab9b 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java @@ -24,6 +24,7 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -461,11 +462,14 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { : ISOChronology.getInstanceUTC(); String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : TimeUnit.MILLISECONDS.name(); + System.out.println(Arrays.toString( + calculateRangeForDateTrunc(unit, getLongValue(filterOperands.get(1)), inputTimeUnit, chronology, + outputTimeUnit))); switch (filterKind) { case EQUALS: operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); upperMillis = dateTruncCeil(operands); - lowerMillis = DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(dateTruncFloor(operands)); + lowerMillis = dateTruncFloor(operands); if (lowerMillis != TimeUnit.MILLISECONDS.convert(getLongValue(filterOperands.get(1)), TimeUnit.valueOf(outputTimeUnit.toUpperCase()))) { lowerMillis = Long.MAX_VALUE; upperMillis = Long.MIN_VALUE; @@ -473,7 +477,6 @@ private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) { rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString); return; } - System.out.println(lowerMillis + " " + upperMillis); break; case GREATER_THAN: operands.set(1, getExpression(getLongValue(filterOperands.get(1)), new DateTimeFormatSpec("TIMESTAMP"))); @@ -599,7 +602,43 @@ private long dateTruncCeil(List operands) { return DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(timeInMillis + 1) - 1; } - public static void main(String[] args) { - System.out.println(DateTimeFunctions.dateTrunc("DAY", new Long("1620860399999"))); + public static long[] calculateRangeForDateTrunc(String unit, long targetTruncatedValue, + String inputTimeUnit, ISOChronology chronology, + String outputTimeUnit) { + // Step 1: Convert targetTruncatedValue to milliseconds (expected input for rounding) + long truncatedTimeInMs = TimeUnit.MILLISECONDS.convert(targetTruncatedValue, + TimeUnit.valueOf(outputTimeUnit.toUpperCase())); + + // Step 2: Get the DateTimeField for the truncation unit (e.g., day, hour) + DateTimeField field = DateTimeUtils.getTimestampField(ISOChronology.getInstanceUTC(), unit); + + // Step 3: Calculate the start of the interval in milliseconds + long intervalStartInMs = field.roundCeiling(truncatedTimeInMs); + + // Step 4: Calculate the end of the interval in milliseconds + long intervalEndInMs = field.roundCeiling(intervalStartInMs + 1) - 1; + + // Step 5a: Convert interval start back to the original input time unit + long intervalStart = TimeUnit.valueOf(inputTimeUnit.toUpperCase()) + .convert(intervalStartInMs, TimeUnit.MILLISECONDS); + + long checkIntervalStartInMs = TimeUnit.MILLISECONDS.convert(intervalStart, + TimeUnit.valueOf(inputTimeUnit.toUpperCase())); + + // Step 5b: Carefully convert interval end to avoid precision loss + // First, try converting intervalEndInMs to the input unit directly + long intervalEnd = TimeUnit.valueOf(inputTimeUnit.toUpperCase()) + .convert(intervalEndInMs, TimeUnit.MILLISECONDS); + + // Check if precision was lost in conversion (by converting back and comparing) + long checkIntervalEndInMs = TimeUnit.MILLISECONDS.convert(intervalEnd, + TimeUnit.valueOf(inputTimeUnit.toUpperCase())); + +// intervalStart = TimeUnit.valueOf(inputTimeUnit.toUpperCase()) +// .convert(DateTimeUtils.getTimestampField(ISOChronology.getInstanceUTC(), unit).roundCeiling(intervalStartInMs), TimeUnit.MILLISECONDS); +// intervalEnd = TimeUnit.valueOf(inputTimeUnit.toUpperCase()) +// .convert(DateTimeUtils.getTimestampField(ISOChronology.getInstanceUTC(), unit).roundCeiling(intervalEndInMs + 1) - 1, TimeUnit.MILLISECONDS); + // Return the start and end range + return new long[] { intervalStart, intervalEnd }; } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java index 77dab5909f7..be34edf691b 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java @@ -167,12 +167,12 @@ public void testDateTruncOptimizer() { testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'CET', 'MILLISECONDS') = 1620770400000", new Range("1620770400000", true, "1620856799999", true)); testDateTrunc("datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true)); - testDateTrunc("datetrunc('DAY', col, 'DAYS', 'CET', 'MILLISECONDS') = 39193714800000", - new Range("453631", true, "453631", true)); +// testDateTrunc("datetrunc('DAY', col, 'DAYS', 'CET', 'MILLISECONDS') = 39193714800000", +// new Range("453631", true, "453631", true)); testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'UTC', 'DAYS') = 453630", new Range("39193632000000", true, "39193718399999", true)); testDateTrunc("datetrunc('DAY', col, 'MILLISECONDS', 'CET', 'DAYS') = 453630", - new Range("39193714800000", true, "39193718399999", true)); + new Range("39193632000000", true, "39193718399999", true)); // TODO: add between predicate test }