Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add DATE_TRUNC Optimizer #14385

Draft
wants to merge 22 commits into
base: master
Choose a base branch
from
Draft

Add DATE_TRUNC Optimizer #14385

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
e3efa65
add floor and ceil func
ashishjayamohan Nov 4, 2024
731b943
Merge branch 'apache:master' into optimize-date-trunc
ashishjayamohan Nov 5, 2024
749bbbb
add date trunc optimization
ashishjayamohan Nov 5, 2024
c2dbe08
fix style
ashishjayamohan Nov 5, 2024
8dda15e
fix range query column expr
ashishjayamohan Nov 5, 2024
2edad2a
add support for between
ashishjayamohan Nov 5, 2024
52490a3
refactor test functions
ashishjayamohan Nov 5, 2024
cb387da
refactor back and add test
ashishjayamohan Nov 5, 2024
0b41d25
add tests for range pred
ashishjayamohan Nov 5, 2024
9391b48
style
ashishjayamohan Nov 5, 2024
36e7639
fix algo and tests
ashishjayamohan Nov 5, 2024
bc08f03
fix between operator
ashishjayamohan Nov 6, 2024
b09fc49
add tests
ashishjayamohan Nov 6, 2024
eec0e7f
use correct output format
ashishjayamohan Nov 6, 2024
614e176
add helper function
ashishjayamohan Nov 6, 2024
624e360
make unit step less brittle
ashishjayamohan Nov 6, 2024
5e87ed7
correct conversion to output unit
ashishjayamohan Nov 6, 2024
4b836c7
add javadocs
ashishjayamohan Nov 6, 2024
470d4bb
fix short circuit
ashishjayamohan Nov 7, 2024
321cc6a
fix range settings
ashishjayamohan Nov 7, 2024
e3a6651
everything but time zones
ashishjayamohan Nov 11, 2024
2e6bb78
just logging for future
ashishjayamohan Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 127 additions & 0 deletions .../main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,23 @@
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.function.DateTimeUtils;
import org.apache.pinot.common.function.TimeZoneKey;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.Literal;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.core.operator.transform.function.DateTimeConversionTransformFunction;
import org.apache.pinot.core.operator.transform.function.DateTruncTransformFunction;
import org.apache.pinot.core.operator.transform.function.TimeConversionTransformFunction;
import org.apache.pinot.spi.data.DateTimeFieldSpec.TimeFormat;
import org.apache.pinot.spi.data.DateTimeFormatSpec;
import org.apache.pinot.spi.data.DateTimeGranularitySpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.sql.FilterKind;
import org.joda.time.chrono.ISOChronology;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -84,6 +88,8 @@ Expression optimize(Expression filterExpression) {
optimizeTimeConvert(filterFunction, filterKind);
} else if (functionName.equalsIgnoreCase(DateTimeConversionTransformFunction.FUNCTION_NAME)) {
optimizeDateTimeConvert(filterFunction, filterKind);
} else if (functionName.equalsIgnoreCase(DateTruncTransformFunction.FUNCTION_NAME)) {
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
optimizeDateTrunc(filterFunction, filterKind);
}
}
}
Expand Down Expand Up @@ -411,6 +417,80 @@ && isStringLiteral(dateTimeConvertOperands.get(3)),
}
}

private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) {
List<Expression> filterOperands = filterFunction.getOperands();
List<Expression> dateTruncOperands = filterOperands.get(0).getFunctionCall().getOperands();

// Check if date trunc function is being applied on a literal value
if (dateTruncOperands.get(1).isSetLiteral()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. I think you want a testInvalidFilterOptimizer unit test for this.
  2. nit: this comment is identical to the code below it. a better comment would say why we can't/don't optimize literal values.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since DATETRUNC can be applied on literals, my thinking was that if we returned at this point, some other optimizer would precompute this value. I need to look into the other optimizers to figure out if this is the case. If this is not applied elsewhere, I'll add the computation for the DATETRUNC of the literal and new query creation here.

return;
}

Long lowerMillis = null;
Long upperMillis = null;
DateTimeFormatSpec inputFormat = new DateTimeFormatSpec("TIMESTAMP");
String inputTimeUnit = (dateTruncOperands.size() >= 3) ? dateTruncOperands.get(2).getLiteral().getStringValue()
: TimeUnit.MILLISECONDS.name();
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
String outputTimeUnit = (dateTruncOperands.size() == 5) ? dateTruncOperands.get(4).getLiteral().getStringValue()
: TimeUnit.MILLISECONDS.name();
boolean lowerInclusive = true;
boolean upperInclusive = true;
List<Expression> operands = new ArrayList<>(dateTruncOperands);
switch (filterKind) {
case EQUALS:
operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit));
lowerMillis = dateTruncFloor(operands);
upperMillis = dateTruncCeil(operands);
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
// Check if it is impossible to obtain literal equality
if (lowerMillis != TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(filterOperands.get(1)),
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
TimeUnit.valueOf(outputTimeUnit))) {
lowerMillis = Long.MAX_VALUE;
upperMillis = Long.MIN_VALUE;
}
break;
case GREATER_THAN:
lowerInclusive = false;
operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit));
lowerMillis = dateTruncCeil(operands);
break;
case GREATER_THAN_OR_EQUAL:
operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit));
lowerInclusive = false;
lowerMillis = dateTruncCeil(operands);
if (dateTruncFloor(operands)
== inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) {
lowerInclusive = true;
lowerMillis = dateTruncFloor(operands);
}
break;
case LESS_THAN:
upperInclusive = false;
operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit));
upperMillis = dateTruncFloor(operands);
if (upperMillis != inputFormat.fromFormatToMillis(getLongValue(filterOperands.get(1)))) {
upperInclusive = true;
upperMillis = dateTruncCeil(operands);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this recomputed here?

}
break;
case LESS_THAN_OR_EQUAL:
operands.set(1, filterOperands.get(1));
upperMillis = dateTruncCeil(operands);
break;
case BETWEEN:
operands.set(1, getExpression(filterOperands.get(1), inputFormat, inputTimeUnit, outputTimeUnit));
operands.set(1, getExpression(filterOperands.get(2), inputFormat, inputTimeUnit, outputTimeUnit));
upperMillis = dateTruncCeil(operands);
break;
default:
throw new IllegalStateException();
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
}
lowerMillis = (lowerMillis == null) ? 0 : lowerMillis;
upperMillis = (upperMillis == null) ? Long.MAX_VALUE : upperMillis;

String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString();
rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString);
}

private boolean isStringLiteral(Expression expression) {
Literal literal = expression.getLiteral();
return literal != null && literal.isSetStringValue();
Expand Down Expand Up @@ -446,4 +526,51 @@ private static void rewriteToRange(Function filterFunction, Expression expressio
newOperands.add(RequestUtils.getLiteralExpression(rangeString));
filterFunction.setOperands(newOperands);
}


private Expression getExpression(Expression value, DateTimeFormatSpec inputFormat,
String inputTimeUnit, String outputTimeUnit) {
Literal literal = new Literal();
if (value.getLiteral().isSetStringValue()) {
literal.setLongValue(inputFormat.fromFormatToMillis(
value.getLiteral().getStringValue()));
} else {
literal.setLongValue(TimeUnit.valueOf(inputTimeUnit).convert(getLongValue(value),
TimeUnit.valueOf(outputTimeUnit)));
}
Expression expression = new Expression(ExpressionType.LITERAL);
expression.setLiteral(literal);
return expression;
}

/**
* Helper function mimicking date trunc function
*/
private long dateTruncFloor(List<Expression> operands) {
String unit = operands.get(0).getLiteral().getStringValue();
long timeValue = getLongValue(operands.get(1));
String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue()
: TimeUnit.MILLISECONDS.name();
ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey
.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC();
String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : inputTimeUnit;
return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit)
.roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))),
TimeUnit.MILLISECONDS);
}

/**
* Helper function that finds the maximum value (ceiling) that truncates to specified value
* Computes ceiling inverse of date trunc function
*/
private long dateTruncCeil(List<Expression> operands) {
String unit = operands.get(0).getLiteral().getStringValue();
String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue()
: TimeUnit.MILLISECONDS.name();
ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey
.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC();
// Add value of 1 unit as specified, subtract one to find maximum value that will truncate to desired value
return dateTruncFloor(operands) + TimeUnit.valueOf(inputTimeUnit).convert(
DateTimeUtils.getTimestampField(chronology, unit).roundCeiling(1), TimeUnit.MILLISECONDS) - 1;
}
}
96 changes: 79 additions & 17 deletions ...t/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,18 @@ public void testTimeConvert() {
new Range(1620777600000L, true, 1620864000000L, false));

// Other input format
testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", new Range(27013846L, false, null, false));
testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", new Range(null, false, 450254L, true));
testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760",
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
new Range(27013846L, false, null, false));
testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286",
new Range(null, false, 450254L, true));
testTimeConvert("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254",
new Range(18759L, false, 18760L, true));
testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759", new Range(1620777600L, true, 1620864000L, false));
testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759",
new Range(1620777600L, true, 1620864000L, false));

// Invalid time
testInvalidTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5");
testInvalidTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760");
testInvalidFilterOptimizer("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5");
testInvalidFilterOptimizer("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760");
}

@Test
Expand Down Expand Up @@ -92,14 +95,17 @@ public void testEpochToEpochDateTimeConvert() {
new Range(1620831600000L, true, 1620833400000L, false));

// Other output format
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')"
+ " > 27013846",
new Range(1620831600000L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') "
+ "< 2701528",
new Range(null, false, 1620918000000L, false));
testTimeConvert(
"dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND "
+ "1620917160", new Range(1620831600000L, true, 1620918000000L, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') "
+ "> 900462",
new Range(1620833400000L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255",
new Range(null, false, 1620918000000L, false));
Expand All @@ -121,23 +127,25 @@ public void testEpochToEpochDateTimeConvert() {
new Range(1620833400L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255",
new Range(null, false, 27015300L, false));
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760",
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') "
+ "BETWEEN 18759 AND 18760",
new Range(18759L, true, 18761L, false));
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759",
new Range(18759L, true, 18760L, false));

// Invalid time
testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5");
testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846");
testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5");
testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846");
}

@Test
public void testSDFToEpochDateTimeConvert() {
testTimeConvert(
"dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false));
testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') < 1620917160000", new Range(null, false, "2021-05-13 15:00:00", false));
testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', "
+ "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false,
"2021-05-13 15:00:00", false));
testTimeConvert(
"dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', '30:MINUTES') "
+ "BETWEEN 1620830760000 AND 1620917160000",
Expand All @@ -147,14 +155,68 @@ public void testSDFToEpochDateTimeConvert() {
+ "1620830760000", new Range("2021-05-12", false, "2021-05-12", true));

// Invalid time
testInvalidTimeConvert(
testInvalidFilterOptimizer(
"dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') > 1620830760000.5");
testInvalidTimeConvert(
testInvalidFilterOptimizer(
"dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') < 1620917160");
}


@Test
public void testDateTruncOptimizer() {
testDateTrunc(
"datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false));
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
testDateTrunc(
"dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false));
testDateTrunc(
"DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true));
testDateTrunc(
"DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true));

testDateTrunc(
"datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true));
testDateTrunc(
"datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true));

testDateTrunc(
"datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"DATETRUNC('DAY', col) > 1620864000000", new Range("1620950399999", false, Long.MAX_VALUE, true));

testDateTrunc(
"datetrunc('DAY', col) >= 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true));

testDateTrunc(
"datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true));
testDateTrunc(
"dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true));

testDateTrunc(
"datetrunc('DAY', col, 'DAYS', 'UTC', 'DAYS') = 453631", new Range("453631", true, "453631", true));
}

/**
* Helper method to test optimizing DATE_TRUNC on the given filter.
*/
private void testDateTrunc(String filterString, Range expectedRange) {
Expression originalExpression = CalciteSqlParser.compileToExpression(filterString);
Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString));
Function function = optimizedFilterExpression.getFunctionCall();
assertEquals(function.getOperator(), FilterKind.RANGE.name());
List<Expression> operands = function.getOperands();
assertEquals(operands.size(), 2);
assertEquals(operands.get(0),
originalExpression.getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(1));
String rangeString = operands.get(1).getLiteral().getStringValue();
assertEquals(rangeString, expectedRange.getRangeString());
}

/**
* Helper method to test no-op TIME_CONVERT filter (same input and output time unit).
*/
Expand Down Expand Up @@ -192,9 +254,9 @@ private void testTimeConvert(String filterString, Range expectedRange) {
}

/**
* Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT with invalid time in filter.
* Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT/DATE_TRUNC with invalid time in filter.
*/
private void testInvalidTimeConvert(String filterString) {
private void testInvalidFilterOptimizer(String filterString) {
Expression originalExpression = CalciteSqlParser.compileToExpression(filterString);
Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString));
assertEquals(optimizedFilterExpression, originalExpression);
Expand Down
Loading