Merge branch 'polardb:main' into main

README.md

@@ -116,7 +116,7 @@ You are welcome to make contributions to PolarDB-X. We appreciate all the contri
 ## Community
 You can join these groups and chats to discuss and ask PolarDB-X related questions:
  - DingTalk Group: [32432897](https://h5.dingtalk.com/circle/healthCheckin.html?dtaction=os&corpId=dingc5456617ca6ab502e1cc01e222598659&1b3d4=1ec1b&cbdbhh=qwertyuiop#/)  
-   ![DingTalk Group](docs/images/dingtalk_group.png)
+   ![DingTalk Group](docs/images/dingtalk_group.jpg)
  - WeChat Group: 阿里云 PolarDB-X 开源交流群 (Contact group manager to get into wechat group. Managers' ID: oldbread3, hustfxj, agapple0002)   
    ![WeChat Manager 1](docs/images/wechat_manager_a.jpg)  ![WeChat Manager 2](docs/images/wechat_manager_b.jpg) ![WeChat Manager 3](docs/images/wechat_manager_c.jpg)
 

codestyle/format_lab.sh

@@ -2,7 +2,7 @@
 
 baseDir=`pwd`/
 commit=`git merge-base origin/polardbx_opensource HEAD`
-files=`git diff --name-only ${commit} | grep '.java' | grep -v 'polardbx-calcite' | grep -v 'polardbx-rpc/src/main/java/com/mysql/cj'| grep -v 'com/alibaba/polardbx/rpc/cdc'| xargs -I {} echo ${baseDir}{}`
+files=`git diff --name-only ${commit} | grep '.java' | grep -v 'polardbx-calcite' | grep -v 'polardbx-orc' | grep -v 'polardbx-rpc/src/main/java/com/mysql/cj'| grep -v 'com/alibaba/polardbx/rpc/cdc'| xargs -I {} echo ${baseDir}{}`
 count=0
 batchFile=''
 for file in $files; do

docs/en/quickstart-development.md

@@ -80,12 +80,13 @@ make install
 
 ```shell
 # install JDK 1.8 and Maven 3
-# enter the polardbx-sql directory
-cd polardbx-sql/
 
 # make sure polardbx-rpc (polardbx-glue) initialized
 git submodule update --init
 
+# enter the polardbx-sql directory 
+cd polardbx-sql/
+
 # compile&install
 mvn install -D maven.test.skip=true -D env=release 
 

docs/zh_CN/quickstart-development.md

@@ -126,7 +126,7 @@ mkdir -p /u01/my3306/{data,log,run,tmp,mysql}
 
 - metadb user：以下采用`my_polarx`
 - metadb database：创建metadb库，以下采用 `polardbx_meta_db_polardbx`
-- 密码加密key（dnPasswordKey)：以下采用 `asdf1234ghjk5678` (如果要修改加密key, 新key的长度要求为16位)
+- 密码加密key（dnPasswordKey)：以下采用 `asdf1234ghjk5678`
 - PolarDB-X默认用户名：默认为 `polarx_root`
 - PolarDB-X默认用户密码：默认为 `123456`，可通过 `-S` 参数修改
 
@@ -199,7 +199,7 @@ mysql -h127.1 -P8527 -upolardbx_root
 - metadb database：和启动PolarDB-X时设置的值保持一致，以下采用 `polardbx_meta_db_polardbx`
 - metadb password：和启动PolarDB-X时设置的值保持一致，需使用密文，以下采用`HMqvkvXZtT7XedA6t2IWY8+D7fJWIJir/mIY1Nf1b58=`
 - metadb port：和启动MySQL时设置的值保持一致，以下采用 `4886`
-- 密码加密key（dnPasswordKey)：和启动PolarDB-X时设置的值保持一致，以下采用 `asdf1234ghjk5678` (如果要修改加密key, 新key的长度要求为16位)
+- 密码加密key（dnPasswordKey)：和启动PolarDB-X时设置的值保持一致，以下采用 `asdf1234ghjk5678`
 - PolarDB-X用户名：和启动PolarDB-X时设置的值保持一致，以下采用默认值 `polardbx_root`
 - PolarDB-X用户密码：和启动PolarDB-X时设置的值保持一致，需使用密文，以下采用默认值`H1AzXc2NmCs61dNjH5nMvA==`
 - PolarDB-X端口：和启动PolarDB-X时设置的值保持一致，以下采用默认值 `8527`

polardbx-calcite/pom.xml

@@ -20,13 +20,13 @@ limitations under the License.
     <parent>
         <artifactId>polardbx</artifactId>
         <groupId>com.alibaba.polardbx</groupId>
-        <version>5.4.18-SNAPSHOT</version>
+        <version>${revision}</version>
     </parent>
     <artifactId>polardbx-calcite</artifactId>
-    <packaging>jar</packaging>
-    <version>5.4.18-SNAPSHOT</version>
+        <packaging>jar</packaging>
+        <version>${calcite.version}</version>
+        <name>${project.artifactId} module for polardbx ${project.version}</name>
     <description>Core Calcite APIs and engine.</description>
-    <name>${project.artifactId} module for polardbx ${project.version}</name>
     <properties>
         <top.dir>${project.basedir}/..</top.dir>
         <build.timestamp>${maven.build.timestamp}</build.timestamp>
@@ -57,10 +57,12 @@ limitations under the License.
       <version>${commons-dbcp.version}</version>
     </dependency>
     <dependency>
-<!--      <groupId>org.apache.commons</groupId>-->
-<!--      <artifactId>commons-lang3</artifactId>-->
-        <groupId>org.apache.commons</groupId>
-        <artifactId>commons-lang3</artifactId>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
@@ -195,6 +197,12 @@ limitations under the License.
       <artifactId>polardbx-rule</artifactId>
       <version>${parent.version}</version>
     </dependency>
+      <dependency>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-inline</artifactId>
+          <version>${mockito.verison}</version>
+          <scope>test</scope>
+      </dependency>
   </dependencies>
 
     <build>

polardbx-calcite/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java

@@ -75,7 +75,7 @@ public JavaTypeFactoryImpl(RelDataTypeSystem typeSystem) {
 
   public RelDataType createStructType(Class type) {
     final List<RelDataTypeField> list = new ArrayList<>();
-    for (Field field : type.getFields()) {
+    for (Field field : type.getDeclaredFields()) {
       if (!Modifier.isStatic(field.getModifiers())) {
         // FIXME: watch out for recursion
         final Type fieldType = fieldType(field);

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java

@@ -19,6 +19,7 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.CorrelationId;
 import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
 import org.apache.calcite.rel.metadata.MetadataFactory;
 import org.apache.calcite.rel.metadata.MetadataFactoryImpl;
 import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -150,19 +151,26 @@ public MetadataFactory getMetadataFactory() {
    * method, then use {@link RelOptRuleCall#getMetadataQuery()} instead. */
   public RelMetadataQuery getMetadataQuery() {
     if (mq == null) {
-      mq = RelMetadataQuery.instance();
+      mq = buildMetaQuery();
     }
 
     RelMetadataQuery local = mq;
     if (local == null) {
       // maybe some concurrent thread call invalidateMetadataQuery
       // cache RelMetadataQuery on the stack to in case of returning null
-      local = RelMetadataQuery.instance();
+      local = buildMetaQuery();
       mq = local;
     }
     return local;
   }
 
+  private RelMetadataQuery buildMetaQuery() {
+    RelMetadataProvider provider = getMetadataProvider();
+    // provider in cluster cannot be a JaninoRelMetadataProvider,
+    // for it doesn't support RelMetadataProvider.apply method
+    return RelMetadataQuery.instance(JaninoRelMetadataProvider.of(provider));
+  }
+
   /**
    * Should be called whenever the current {@link RelMetadataQuery} becomes
    * invalid. Typically invoked from {@link RelOptRuleCall#transformTo}.

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelOptUtil.java

@@ -96,12 +96,14 @@
 import org.apache.calcite.runtime.CalciteContextException;
 import org.apache.calcite.runtime.PredicateImpl;
 import org.apache.calcite.schema.ModifiableView;
+import org.apache.calcite.sql.SqlBasicCall;
 import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlLiteral;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlSelect;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.MultisetSqlType;
 import org.apache.calcite.sql.type.SqlTypeName;
@@ -235,7 +237,7 @@ public static Set<RelOptTable> findTables(RelNode rel) {
    */
   public static List<RelOptTable> findAllTables(RelNode rel) {
     final Multimap<Class<? extends RelNode>, RelNode> nodes =
-        RelMetadataQuery.instance().getNodeTypes(rel);
+       rel.getCluster().getMetadataQuery().getNodeTypes(rel);
     final List<RelOptTable> usedTables = new ArrayList<>();
     for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) {
       if (TableScan.class.isAssignableFrom(e.getKey())) {
@@ -259,7 +261,7 @@ public static Set<RelNode> findCorrelates(RelNode rel) {
    */
   public static List<RelNode> findAllCorrelate(RelNode rel) {
     final Multimap<Class<? extends RelNode>, RelNode> nodes =
-        RelMetadataQuery.instance().getNodeTypes(rel);
+        rel.getCluster().getMetadataQuery().getNodeTypes(rel);
     final List<RelNode> usedTables = new ArrayList<>();
     for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) {
       if (Correlate.class.isAssignableFrom(e.getKey())) {
@@ -1925,6 +1927,11 @@ public static boolean eq(
       return litmus.succeed();
     }
 
+    if (type1.getSqlTypeName() == SqlTypeName.DECIMAL
+        && type2.getSqlTypeName() ==  SqlTypeName.DECIMAL) {
+      return litmus.succeed();
+    }
+
     if (type1 != type2) {
       return litmus.fail("type mismatch:\n{}:\n{}\n{}:\n{}",
           desc1, type1.getFullTypeString(),
@@ -2877,7 +2884,7 @@ public static boolean containsLimit(RelNode ancestor) {
     try {
       new RelVisitor() {
         public void visit(RelNode node, int ordinal, RelNode parent) {
-          if (node instanceof Sort) {
+          if (node instanceof Sort&&((Sort)node).withLimit()) {
             throw Util.FoundOne.NULL;
           }
           super.visit(node, ordinal, parent);
@@ -4349,6 +4356,26 @@ public boolean test(RelNode relNode) {
       return traits.getCollation().isTop() && traits.getDistribution().isTop();
     }
   };
+
+  public static boolean isUnion(SqlNode node) {
+    return node instanceof SqlBasicCall && node.getKind() == SqlKind.UNION;
+  }
+
+  public static int getColumnCount(SqlNode node) {
+    if (isUnion(node)) {
+      final SqlNode[] children = ((SqlBasicCall) node).getOperands();
+      for (int i = 0; i < children.length; i++) {
+        final int columnCount = getColumnCount(children[i]);
+        if (columnCount > 0) {
+          return columnCount;
+        }
+      }
+      return -1;
+    } else if (node instanceof SqlSelect) {
+      return ((SqlSelect) node).getSelectList().size();
+    }
+    return -1;
+  }
 }
 
 // End RelOptUtil.java

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelTraitSet.java

@@ -20,6 +20,9 @@
 import org.apache.calcite.rel.RelCollationTraitDef;
 import org.apache.calcite.rel.RelDistribution;
 import org.apache.calcite.rel.RelDistributionTraitDef;
+import org.apache.calcite.rel.RelPartitionWise;
+import org.apache.calcite.rel.RelPartitionWiseTraitDef;
+import org.apache.calcite.rel.RelPartitionWises;
 import org.apache.calcite.util.mapping.Mappings;
 
 import com.google.common.collect.ImmutableList;
@@ -390,6 +393,10 @@ public RelTraitSet getDefaultSansConvention() {
     return (@Nullable T) getTrait(RelCollationTraitDef.INSTANCE);
   }
 
+  public <T extends RelPartitionWise> @Nullable T getPartitionWise() {
+    return (@Nullable T) getTrait(RelPartitionWiseTraitDef.INSTANCE);
+  }
+
   /**
    * Returns the size of the RelTraitSet.
    *

polardbx-calcite/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java

@@ -173,6 +173,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
 
   boolean enableDerive = true;
 
+  boolean enableColumnar = false;
+
   /**
    * Extra roots for explorations.
    */
@@ -401,7 +403,7 @@ protected void registerMaterializations() {
     this.root = null;
     this.originalRoot = null;
     if (ruleCounter != null) {
-      ruleCount = ruleCounter.values().stream().reduce(0L, (a,b) -> a + b).longValue();
+      ruleCount = ruleCounter.values().stream().reduce(0L, Long::sum);
       this.ruleCounter.clear();
     }
   }
@@ -1720,7 +1722,7 @@ public RelOptCost getStartUpCost(RelNode rel, RelMetadataQuery mq) {
       return costFactory.makeInfiniteCost();
     }
     // use new mq to avoid metadata cache
-    mq = RelMetadataQuery.instance();
+    mq = RelMetadataQuery.instance(mq.metadataProvider);
     // for efficiency, use RelSubsetBest Cost For CumulativeCost
     mq.setUseRelSubsetBestCostForCumulativeCost(true);
     RelOptCost cost = mq.getStartUpCost(rel);
@@ -1765,4 +1767,12 @@ public boolean isEnableDerive() {
   public void setEnableDerive(boolean enableDerive) {
     this.enableDerive = enableDerive;
   }
+
+  public boolean isEnableColumnar() {
+    return enableColumnar;
+  }
+
+  public void setEnableColumnar(boolean enableColumnar) {
+    this.enableColumnar = enableColumnar;
+  }
 }

polardbx-calcite/src/main/java/org/apache/calcite/rel/AbstractRelNode.java

@@ -321,7 +321,6 @@ public RelNode onRegister(RelOptPlanner planner) {
       r = copy(getTraitSet(), inputs);
     }
     r.recomputeDigest();
-    assert r.isValid(Litmus.THROW, null);
     return r;
   }
 

polardbx-calcite/src/main/java/org/apache/calcite/rel/RelDistribution.java

@@ -48,6 +48,10 @@ public interface RelDistribution extends RelMultipleTrait {
    */
   @Nonnull List<Integer> getKeys();
 
+  boolean isShardWise();
+
+  @Nonnull Integer getShardCnt();
+
   RelDistribution apply(Mappings.TargetMapping mapping);
 
   /** Type of distribution. */