HIVE-29339: Remove DISTINCT indicator from SqlAggFunctions

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -954,7 +954,7 @@ public static AggregateCall createSingleArgAggCall(String funcName, RelOptCluste
       PrimitiveTypeInfo typeInfo, Integer pos, RelDataType aggFnRetType) {
     ImmutableList.Builder<RelDataType> aggArgRelDTBldr = new ImmutableList.Builder<RelDataType>();
     aggArgRelDTBldr.add(TypeConverter.convert(typeInfo, cluster.getTypeFactory()));
-    SqlAggFunction aggFunction = SqlFunctionConverter.getCalciteAggFn(funcName, false,
+    SqlAggFunction aggFunction = SqlFunctionConverter.getCalciteAggFn(funcName,
         aggArgRelDTBldr.build(), aggFnRetType);
     List<Integer> argList = new ArrayList<Integer>();
     argList.add(pos);

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlAverageAggFunction.java

@@ -22,15 +22,15 @@
 import org.apache.calcite.sql.SqlFunctionCategory;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlSplittableAggFunction;
+import org.apache.calcite.sql.SqlSyntax;
 import org.apache.calcite.sql.type.SqlOperandTypeChecker;
 import org.apache.calcite.sql.type.SqlOperandTypeInference;
 import org.apache.calcite.sql.type.SqlReturnTypeInference;
 import org.apache.calcite.util.Optionality;
 
-public class HiveSqlAverageAggFunction extends SqlAggFunction implements CanAggregateDistinct {
-  private final boolean isDistinct;
+public class HiveSqlAverageAggFunction extends SqlAggFunction {
 
-  public HiveSqlAverageAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference,
+  public HiveSqlAverageAggFunction(SqlReturnTypeInference returnTypeInference,
       SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) {
     super(
         "avg",
@@ -43,7 +43,6 @@ public HiveSqlAverageAggFunction(boolean isDistinct, SqlReturnTypeInference retu
         false,
         false,
         Optionality.FORBIDDEN);
-    this.isDistinct = isDistinct;
   }
 
   @Override
@@ -55,7 +54,7 @@ public <T> T unwrap(Class<T> clazz) {
   }
 
   @Override
-  public boolean isDistinct() {
-    return isDistinct;
+  public SqlSyntax getSyntax() {
+    return SqlSyntax.FUNCTION_STAR;
   }
 }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java

@@ -41,14 +41,12 @@
 
 import com.google.common.collect.ImmutableList;
 
-public class HiveSqlCountAggFunction extends SqlAggFunction implements CanAggregateDistinct {
-
-  final boolean                isDistinct;
+public class HiveSqlCountAggFunction extends SqlAggFunction {
   final SqlReturnTypeInference returnTypeInference;
   final SqlOperandTypeInference operandTypeInference;
   final SqlOperandTypeChecker operandTypeChecker;
 
-  public HiveSqlCountAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference,
+  public HiveSqlCountAggFunction(SqlReturnTypeInference returnTypeInference,
       SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) {
     super(
         "count",
@@ -57,17 +55,11 @@ public HiveSqlCountAggFunction(boolean isDistinct, SqlReturnTypeInference return
         operandTypeInference,
         operandTypeChecker,
         SqlFunctionCategory.NUMERIC);
-    this.isDistinct = isDistinct;
     this.returnTypeInference = returnTypeInference;
     this.operandTypeChecker = operandTypeChecker;
     this.operandTypeInference = operandTypeInference;
   }
 
-  @Override
-  public boolean isDistinct() {
-    return isDistinct;
-  }
-
   @Override
   public SqlSyntax getSyntax() {
     return SqlSyntax.FUNCTION_STAR;
@@ -91,7 +83,7 @@ class HiveCountSplitter extends CountSplitter {
     @Override
     public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) {
       return AggregateCall.create(
-          new HiveSqlCountAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker),
+          new HiveSqlCountAggFunction(returnTypeInference, operandTypeInference, operandTypeChecker),
           false, ImmutableIntList.of(), -1,
           typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true), "count");
     }
@@ -122,14 +114,14 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
       }
       int ordinal = extra.register(node);
       return AggregateCall.create(
-          new HiveSqlSumEmptyIsZeroAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker),
+          new HiveSqlSumEmptyIsZeroAggFunction(returnTypeInference, operandTypeInference, operandTypeChecker),
           false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name);
     }
   }
 
   @Override
   public @Nullable SqlAggFunction getRollup() {
-    return new HiveSqlSumEmptyIsZeroAggFunction(isDistinct(), getReturnTypeInference(), getOperandTypeInference(),
+    return new HiveSqlSumEmptyIsZeroAggFunction(getReturnTypeInference(), getOperandTypeInference(),
         getOperandTypeChecker());
   }
 }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumAggFunction.java

@@ -30,6 +30,7 @@
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlSplittableAggFunction;
 import org.apache.calcite.sql.SqlSplittableAggFunction.SumSplitter;
+import org.apache.calcite.sql.SqlSyntax;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.ReturnTypes;
 import org.apache.calcite.sql.type.SqlOperandTypeChecker;
@@ -46,15 +47,14 @@
  * <code>long</code>, <code>float</code>, <code>double</code>), and the result
  * is the same type.
  */
-public class HiveSqlSumAggFunction extends SqlAggFunction implements CanAggregateDistinct{
-  final boolean isDistinct;
+public class HiveSqlSumAggFunction extends SqlAggFunction {
   final SqlReturnTypeInference returnTypeInference;
   final SqlOperandTypeInference operandTypeInference;
   final SqlOperandTypeChecker operandTypeChecker;
 
   //~ Constructors -----------------------------------------------------------
 
-  public HiveSqlSumAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference,
+  public HiveSqlSumAggFunction(SqlReturnTypeInference returnTypeInference,
     SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) {
     super(
         "sum",
@@ -66,14 +66,9 @@ public HiveSqlSumAggFunction(boolean isDistinct, SqlReturnTypeInference returnTy
     this.returnTypeInference = returnTypeInference;
     this.operandTypeChecker = operandTypeChecker;
     this.operandTypeInference = operandTypeInference;
-    this.isDistinct = isDistinct;
   }
 
   //~ Methods ----------------------------------------------------------------
-  @Override
-  public boolean isDistinct() {
-    return isDistinct;
-  }
 
   @Override
   public <T> T unwrap(Class<T> clazz) {
@@ -89,7 +84,7 @@ class HiveSumSplitter extends SumSplitter {
     public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) {
       RelDataType countRetType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true);
       return AggregateCall.create(
-        new HiveSqlCountAggFunction(isDistinct, ReturnTypes.explicit(countRetType), operandTypeInference, operandTypeChecker),
+        new HiveSqlCountAggFunction(ReturnTypes.explicit(countRetType), operandTypeInference, operandTypeChecker),
         false, ImmutableIntList.of(), -1, countRetType, "count");
     }
 
@@ -120,11 +115,16 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
         throw new AssertionError("unexpected count " + merges);
       }
       int ordinal = extra.register(node);
-      return AggregateCall.create(new HiveSqlSumAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker),
+      return AggregateCall.create(new HiveSqlSumAggFunction(returnTypeInference, operandTypeInference, operandTypeChecker),
           false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name);
     }
   }
 
+  @Override
+  public SqlSyntax getSyntax() {
+    return SqlSyntax.FUNCTION_STAR;
+  }
+
   @Override
   public SqlAggFunction getRollup() {
     return this;

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java

@@ -42,7 +42,7 @@
 public class HiveSqlSumEmptyIsZeroAggFunction extends SqlAggFunction {
   //~ Constructors -----------------------------------------------------------
 
-  public HiveSqlSumEmptyIsZeroAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference,
+  public HiveSqlSumEmptyIsZeroAggFunction(SqlReturnTypeInference returnTypeInference,
           SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) {
     super("$SUM0",
         null,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateReduceFunctionsRule.java

@@ -300,7 +300,6 @@ private RexNode reduceSum0(
     final AggregateCall sumCall =
         AggregateCall.create(
             new HiveSqlSumAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(sumReturnType),
                 oldCall.getAggregation().getOperandTypeInference(),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.SUM,
@@ -346,7 +345,6 @@ private RexNode reduceAvg(
     final AggregateCall sumCall =
         AggregateCall.create(
             new HiveSqlSumAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(sumReturnType),
                 oldCall.getAggregation().getOperandTypeInference(),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.SUM,
@@ -364,7 +362,6 @@ private RexNode reduceAvg(
     final AggregateCall countCall =
         AggregateCall.create(
             new HiveSqlCountAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(countRetType),
                 oldCall.getAggregation().getOperandTypeInference(),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.COUNT,
@@ -445,7 +442,6 @@ private RexNode reduceStddev(
     final AggregateCall sumArgSquaredAggCall =
         createAggregateCallWithBinding(typeFactory,
             new HiveSqlSumAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(sumSquaredReturnType),
                 InferTypes.explicit(Collections.singletonList(argSquared.getType())),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.SUM,
@@ -461,7 +457,6 @@ private RexNode reduceStddev(
     final AggregateCall sumArgAggCall =
         AggregateCall.create(
             new HiveSqlSumAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(sumReturnType),
                 InferTypes.explicit(Collections.singletonList(argRef.getType())),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.SUM,
@@ -490,7 +485,6 @@ private RexNode reduceStddev(
     final AggregateCall countArgAggCall =
         AggregateCall.create(
             new HiveSqlCountAggFunction(
-                oldCall.isDistinct(),
                 ReturnTypes.explicit(countRetType),
                 oldCall.getAggregation().getOperandTypeInference(),
                 oldCall.getAggregation().getOperandTypeChecker()), //SqlStdOperatorTable.COUNT,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingLastValueRewrite.java

@@ -36,8 +36,8 @@
 import org.apache.calcite.rex.RexWindow;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.fun.SqlBasicAggFunction;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
 
 /**
  * Rule to rewrite a window function containing a last value clause.
@@ -103,9 +103,8 @@ public RexNode visitOver(RexOver over) {
           newOrderKeys.add(new RexFieldCollation(orderKey.left, flags));
         }
         SqlAggFunction s = (SqlAggFunction) over.op;
-        SqlFunctionConverter.CalciteUDAF newSqlAggFunction = new SqlFunctionConverter.CalciteUDAF(
-            over.isDistinct(), FIRST_VALUE_FUNC, s.getReturnTypeInference(), s.getOperandTypeInference(),
-            s.getOperandTypeChecker());
+        SqlAggFunction newSqlAggFunction = SqlBasicAggFunction.create(
+            FIRST_VALUE_FUNC, SqlKind.OTHER_FUNCTION, s.getReturnTypeInference(), s.getOperandTypeChecker());
         List<RexNode> clonedOperands = visitList(over.operands, new boolean[] {false});
         RexWindow window = visitWindow(over.getWindow());
         return rexBuilder.makeOver(over.type, newSqlAggFunction, clonedOperands,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCAggregationPushDownRule.java

@@ -62,8 +62,7 @@ public boolean matches(RelOptRuleCall call) {
       SqlAggFunction f = relOptRuleOperand.getAggregation();
       if (f instanceof HiveSqlCountAggFunction) {
         //count distinct with more that one argument is not supported
-        HiveSqlCountAggFunction countAgg = (HiveSqlCountAggFunction)f;
-        if (countAgg.isDistinct() && 1 < relOptRuleOperand.getArgList().size()) {
+        if (relOptRuleOperand.isDistinct() && 1 < relOptRuleOperand.getArgList().size()) {
           return false;
         }
       }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java

@@ -66,6 +66,7 @@
 import org.apache.calcite.rex.RexWindowBound;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlSyntax;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.ImmutableBitSet;
@@ -1000,7 +1001,18 @@ public ASTNode visitOver(RexOver over) {
       }
 
       // 1. Translate the UDAF
-      final ASTNode wUDAFAst = visitCall(over);
+      ASTNode wUDAFAst = ASTBuilder.createAST(HiveParser.TOK_FUNCTION, "TOK_FUNCTION");
+      if (over.getOperands().isEmpty() && over.op.getSyntax() == SqlSyntax.FUNCTION_STAR) {
+        wUDAFAst = ASTBuilder.createAST(HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR");
+      }
+      if (over.isDistinct()) {
+        wUDAFAst = ASTBuilder.createAST(HiveParser.TOK_FUNCTIONDI, "TOK_FUNCTIONDI");
+      }
+      wUDAFAst.addChild(ASTBuilder.createAST(HiveParser.Identifier, over.op.getName()));
+      wUDAFAst.setTypeInfo(TypeConverter.convert(over.type));
+      for (RexNode operand : over.getOperands()) {
+        wUDAFAst.addChild(operand.accept(this));
+      }
 
       // 2. Add TOK_WINDOW as child of UDAF
       ASTNode wSpec = ASTBuilder.createAST(HiveParser.TOK_WINDOWSPEC, "TOK_WINDOWSPEC");

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java

@@ -493,8 +493,7 @@ private static void replaceEmptyGroupAggr(final RelNode rel, RelNode parent) {
     RelDataType longType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, typeFactory);
     RelDataType intType = TypeConverter.convert(TypeInfoFactory.intTypeInfo, typeFactory);
     // Create the dummy aggregation.
-    SqlAggFunction countFn = SqlFunctionConverter.getCalciteAggFn("count", false,
-        ImmutableList.of(intType), longType);
+    SqlAggFunction countFn = SqlFunctionConverter.getCalciteAggFn("count", ImmutableList.of(intType), longType);
     // TODO: Using 0 might be wrong; might need to walk down to find the
     // proper index of a dummy.
     List<Integer> argList = ImmutableList.of(0);