[CARBONDATA-4138] Reordering Carbon Expression instead of Spark Filter

core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java

@@ -2602,6 +2602,11 @@ private CarbonCommonConstants() {
 
   public static final String FILE_HEADER = "fileHeader";
 
+  @CarbonProperty(dynamicConfigurable = true)
+  public static final String CARBON_OPTIMIZE_FILTER = "carbon.optimize.filter";
+
+  public static final String CARBON_OPTIMIZE_FILTER_DEFAULT = "true";
+
   @CarbonProperty(dynamicConfigurable = true)
   public static final String CARBON_REORDER_FILTER = "carbon.reorder.filter";
 

core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java

@@ -59,6 +59,6 @@ public String getString() {
 
   @Override
   public String getStatement() {
-    return "(" + left.getString() + " or " + right.getString() + ")";
+    return "(" + left.getStatement() + " or " + right.getStatement() + ")";
   }
 }

core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/ExpressionOptimizer.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.optimize;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.optimize.reorder.ExpressionReorder;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * optimize Carbon Expression
+ */
+public class ExpressionOptimizer {
+
+  private final OptimizeRule[] rules = { new ExpressionReorder() };
+
+  public static Expression optimize(CarbonTable table, Expression expression) {
+    if (!CarbonProperties.isFilterOptimizeEnabled()) {
+      return expression;
+    }
+    for (OptimizeRule rule : ExpressionOptimizerHandler.INSTANCE.rules) {
+      expression = rule.optimize(table, expression);
+    }
+    return expression;
+  }
+
+  private static class ExpressionOptimizerHandler {
+    private static final ExpressionOptimizer INSTANCE = new ExpressionOptimizer();
+  }
+}

core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/OptimizeRule.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.optimize;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.scan.expression.Expression;
+
+/**
+ * the base rule of ExpressionOptimizer
+ */
+public abstract class OptimizeRule {
+
+  public abstract Expression optimize(CarbonTable table, Expression expression);
+
+}

core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/reorder/AndMultiExpression.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.optimize.reorder;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+
+/**
+ * new And expression with multiple children (maybe more than two children).
+ */
+public class AndMultiExpression extends MultiExpression {
+
+  @Override
+  public boolean canMerge(Expression child) {
+    return child instanceof AndExpression;
+  }
+
+  @Override
+  public Expression toExpression() {
+    return children.stream()
+        .map(StorageOrdinal::toExpression)
+        .reduce(AndExpression::new)
+        .orElse(null);
+  }
+}

core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/reorder/ExpressionReorder.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.optimize.reorder;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.optimize.OptimizeRule;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * reorder Expression by storage order
+ */
+public class ExpressionReorder extends OptimizeRule {
+
+  @Override
+  public Expression optimize(CarbonTable table, Expression expression) {
+    if (!CarbonProperties.isFilterReorderingEnabled()) {
+      return expression;
+    }
+    MultiExpression multiExpression = MultiExpression.build(expression);
+    // unsupported expression
+    if (multiExpression == null) {
+      return expression;
+    }
+    // remove redundancy filter
+    multiExpression.removeRedundant();
+    // combine multiple filters to single filter
+    multiExpression.combine();
+    // reorder Expression by storage ordinal of columns
+    multiExpression.updateMinOrdinal(columnMapOrdinal(table));
+    multiExpression.sortChildrenByOrdinal();
+    return multiExpression.toExpression();
+  }
+
+  private Map<String, Integer> columnMapOrdinal(CarbonTable table) {
+    List<CarbonColumn> createOrderColumns = table.getCreateOrderColumn();
+    Map<String, Integer> nameMapOrdinal = new HashMap<>(createOrderColumns.size());
+    int dimensionCount = table.getAllDimensions().size();
+    for (CarbonColumn column : createOrderColumns) {
+      if (column.isDimension()) {
+        nameMapOrdinal.put(column.getColName(), column.getOrdinal());
+      } else {
+        nameMapOrdinal.put(column.getColName(), dimensionCount + column.getOrdinal());
+      }
+    }
+    return nameMapOrdinal;
+  }
+}

core/src/main/java/org/apache/carbondata/core/scan/expression/optimize/reorder/ExpressionWithOrdinal.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.optimize.reorder;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.UnknownExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+
+/**
+ * a wrapper class of Expression with storage ordinal
+ */
+public class ExpressionWithOrdinal extends StorageOrdinal {
+  protected Expression expression;
+
+  public ExpressionWithOrdinal(Expression expression) {
+    this.minOrdinal = Integer.MAX_VALUE;
+    this.expression = expression;
+  }
+
+  @Override
+  public void updateMinOrdinal(Map<String, Integer> columnMapOrdinal) {
+    updateMinOrdinal(expression, columnMapOrdinal);
+  }
+
+  private void updateMinOrdinal(Expression expression, Map<String, Integer> nameMapOrdinal) {
+    if (expression != null && expression.getChildren() != null) {
+      if (expression.getChildren().size() == 0) {
+        if (expression instanceof ConditionalExpression) {
+          List<ColumnExpression> columnList =
+              ((ConditionalExpression) expression).getColumnList();
+          for (ColumnExpression columnExpression : columnList) {
+            updateMinOrdinal(columnExpression.getColumnName(), nameMapOrdinal);
+          }
+        }
+      } else {
+        for (Expression subExpression : expression.getChildren()) {
+          if (subExpression instanceof ColumnExpression) {
+            updateMinOrdinal(((ColumnExpression) subExpression).getColumnName(), nameMapOrdinal);
+          } else if (expression instanceof UnknownExpression) {
+            UnknownExpression exp = ((UnknownExpression) expression);
+            List<ColumnExpression> listOfColExpression = exp.getAllColumnList();
+            for (ColumnExpression columnExpression : listOfColExpression) {
+              updateMinOrdinal(columnExpression.getColumnName(), nameMapOrdinal);
+            }
+          } else {
+            updateMinOrdinal(subExpression, nameMapOrdinal);
+          }
+        }
+      }
+    }
+  }
+
+  private void updateMinOrdinal(String columnName, Map<String, Integer> nameMapOrdinal) {
+    Integer ordinal = nameMapOrdinal.get(columnName.toLowerCase());
+    if (ordinal != null && ordinal < minOrdinal) {
+      minOrdinal = ordinal;
+    }
+  }
+
+  @Override
+  public Expression toExpression() {
+    return expression;
+  }
+}