diff --git a/presto-common/pom.xml b/presto-common/pom.xml index 07e80d39313e7..cbc4eea221a3c 100644 --- a/presto-common/pom.xml +++ b/presto-common/pom.xml @@ -78,7 +78,6 @@ com.google.guava guava - test diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java b/presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java rename to presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java index dbcc9d46ee587..5d2b1b11380dd 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.type; +package com.facebook.presto.common.type; import com.facebook.presto.common.block.Block; import com.facebook.presto.common.block.BlockBuilder; @@ -19,9 +19,6 @@ import com.facebook.presto.common.block.Int128ArrayBlockBuilder; import com.facebook.presto.common.block.PageBuilderStatus; import com.facebook.presto.common.function.SqlFunctionProperties; -import com.facebook.presto.common.type.AbstractPrimitiveType; -import com.facebook.presto.common.type.FixedWidthType; -import com.facebook.presto.common.type.StandardTypes; import com.google.common.net.InetAddresses; import io.airlift.slice.Slice; import io.airlift.slice.Slices; diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java b/presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java rename to presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java index 9381ff5c0ea34..dbf624672f903 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.type; +package com.facebook.presto.common.type; import com.facebook.presto.common.block.Block; import com.facebook.presto.common.block.BlockBuilder; @@ -19,9 +19,6 @@ import com.facebook.presto.common.block.PageBuilderStatus; import com.facebook.presto.common.block.VariableWidthBlockBuilder; import com.facebook.presto.common.function.SqlFunctionProperties; -import com.facebook.presto.common.type.AbstractPrimitiveType; -import com.facebook.presto.common.type.FixedWidthType; -import com.facebook.presto.common.type.StandardTypes; import com.google.common.net.InetAddresses; import io.airlift.slice.Slice; import io.airlift.slice.XxHash64; diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java b/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java index cba986d84bcff..fc04b944aaebc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java @@ -105,14 +105,26 @@ private PlanNodeStatsEstimate estimateExpressionNotEqualToLiteral( else { filterRange = new StatisticRange(NEGATIVE_INFINITY, true, POSITIVE_INFINITY, true, 1); } - double filterFactor = 1 - calculateFilterFactor(expressionStatistics, filterRange); + + double filterFactor; + double expressionNDV = expressionStatistics.getDistinctValuesCount(); + if (Double.compare(expressionNDV, 1D) == 0) { + // It's hard to make a meaningful estimate when we have only one distinct value + filterFactor = UNKNOWN_FILTER_COEFFICIENT; + } + else { + filterFactor = 1 - calculateFilterFactor(expressionStatistics, filterRange); + } PlanNodeStatsEstimate.Builder estimate = PlanNodeStatsEstimate.buildFrom(inputStatistics); estimate.setOutputRowCount(filterFactor * (1 - expressionStatistics.getNullsFraction()) * inputStatistics.getOutputRowCount()); if (expressionVariable.isPresent()) { + // If the original NDV was 1, we do not make any changes to the new estimate, since we're not sure if we eliminated the only distinct value + // Otherwise, we reduce the NDV by 1 (unless it was already 0) + double newNDV = Double.compare(expressionNDV, 1D) == 0 ? 1 : max(expressionNDV - 1, 0); VariableStatsEstimate symbolNewEstimate = buildFrom(expressionStatistics) .setNullsFraction(0.0) - .setDistinctValuesCount(max(expressionStatistics.getDistinctValuesCount() - 1, 0)) + .setDistinctValuesCount(newNDV) .build(); estimate = estimate.addVariableStatistics(expressionVariable.get(), symbolNewEstimate); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index d01c5b6ba68f2..37fb1abba807f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -91,6 +91,7 @@ import static java.lang.Double.NaN; import static java.lang.Double.isInfinite; import static java.lang.Double.isNaN; +import static java.lang.Double.max; import static java.lang.Double.min; import static java.lang.String.format; import static java.util.Collections.emptyMap; @@ -98,6 +99,13 @@ public class FilterStatsCalculator { + /** + * + * This value applies a filter factor to upper-bound the size of the variable range selected for an IN predicate + * Since the estimator sums up the individual estimates, we don't want to go beyond 1.0 + * This also impacts NOT IN similarly, we never apply a filter factor of 0.0 for a NOT IN clause + */ + static final double CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT = 0.8; static final double UNKNOWN_FILTER_COEFFICIENT = 0.9; private final Metadata metadata; @@ -403,9 +411,11 @@ protected PlanNodeStatsEstimate visitInPredicate(InPredicate node, Void context) } double notNullValuesBeforeIn = input.getOutputRowCount() * (1 - valueStats.getNullsFraction()); + double ceiledInEstimated = max(notNullValuesBeforeIn * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT, 1.0); + double inEstimateRowCount = min(inEstimate.getOutputRowCount(), ceiledInEstimated); PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.buildFrom(input); - result.setOutputRowCount(min(inEstimate.getOutputRowCount(), notNullValuesBeforeIn)); + result.setOutputRowCount(inEstimateRowCount); if (node.getValue() instanceof SymbolReference) { VariableReferenceExpression valueVariable = toVariable(node.getValue()); @@ -774,9 +784,11 @@ private PlanNodeStatsEstimate estimateIn(RowExpression value, List() diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java index 3b6355bdfd317..ac0b7591ab38b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java @@ -101,6 +101,7 @@ import static com.facebook.presto.sql.MaterializedViewUtils.COUNT; import static com.facebook.presto.sql.MaterializedViewUtils.NON_ASSOCIATIVE_REWRITE_FUNCTIONS; import static com.facebook.presto.sql.MaterializedViewUtils.SUM; +import static com.facebook.presto.sql.MaterializedViewUtils.resolveTableName; import static com.facebook.presto.sql.analyzer.MaterializedViewInformationExtractor.MaterializedViewInfo; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_TABLE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; @@ -641,7 +642,8 @@ protected Node visitAliasedRelation(AliasedRelation node, Void context) @Override protected Node visitRelation(Relation node, Void context) { - if (materializedViewInfo.getBaseTable().isPresent() && node.equals(materializedViewInfo.getBaseTable().get())) { + if (materializedViewInfo.getBaseTable().isPresent() && resolveTableName(node, session, metadata) + .equals(resolveTableName(materializedViewInfo.getBaseTable().get(), session, metadata))) { return materializedView; } throw new IllegalStateException("Mismatching table or non-supporting relation format in base query"); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java index 29217e5451577..a7755d49e6148 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java @@ -41,8 +41,8 @@ import java.util.Objects; import java.util.Optional; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressOperators.java b/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressOperators.java index 5d68bb96e5195..108f27b820140 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressOperators.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressOperators.java @@ -42,9 +42,9 @@ import static com.facebook.presto.common.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.common.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.common.function.OperatorType.XX_HASH_64; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; import static java.lang.System.arraycopy; diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixOperators.java b/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixOperators.java index 44f23753ea08e..029e5b27a892b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixOperators.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixOperators.java @@ -42,10 +42,10 @@ import static com.facebook.presto.common.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.common.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.common.function.OperatorType.XX_HASH_64; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; import static java.lang.Math.max; diff --git a/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java b/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java index aef0e39f5629d..4efe15d47bdf4 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java +++ b/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Optional; @@ -37,6 +38,7 @@ import static java.lang.Double.POSITIVE_INFINITY; import static java.lang.String.format; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; public abstract class AbstractTestFilterStatsCalculator { @@ -63,6 +65,36 @@ public AbstractTestFilterStatsCalculator(boolean withHistograms) .build(); } + /** + * Returns expressions on a variable with NDV 1 and the expected row count after applying the filter + * Row count for the input plan node is assumed to be 100 + * @return + */ + @DataProvider + public static Object[][] ndv1Expressions() + { + return new Object[][] { + {"name <> 'bar'", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + {"name <> 'name' AND name <> 'bar'", 81D}, // 100 * UNKNOWN_FILTER_COEFFICIENT * UNKNOWN_FILTER_COEFFICIENT + {"name <> 'foo' OR name is NULL", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + {"name is NULL OR name <> 'foo'", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + }; + } + + @DataProvider + public static Object[][] inList() + { + return new Object[][] { + {"'one'"}, + {"'one','two'"}, + {"'one','two','three'"}, + {"'one','two','three','four'"}, + {"'one','two','three','four','five'"}, + {"'one','two','three','four','five','six'"}, + {"'one','two','three','four','five','six', 'seven'"} + }; + } + @BeforeClass public void setUp() throws Exception @@ -500,6 +532,50 @@ public void testSymbolEqualsSameSymbolFilter() .build()); } + @Test(dataProvider = "inList") + public void testInPredicateWithoutNDV(String inList) + { + Expression exp = expression("status IN (" + inList + ")"); + TypeProvider customTypes = TypeProvider.fromVariables(ImmutableList.builder() + .add(new VariableReferenceExpression(Optional.empty(), "status", MEDIUM_VARCHAR_TYPE)) + .build()); + + RowExpression rowExpression = translator.translateAndOptimize(exp, customTypes); + + VariableStatsEstimate nameStats = VariableStatsEstimate.builder() + // Nulls fraction is known, but NDV is not. Stats propagation should work + .setNullsFraction(0.0D) + .build(); + + PlanNodeStatsEstimate inputStats = PlanNodeStatsEstimate.builder() + .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "status", MEDIUM_VARCHAR_TYPE), nameStats) + .setOutputRowCount(100D) + .build(); + + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(inputStats, rowExpression, session); + + // The IN filter should always apply a filter factor between (0,1) (never NaN/0/1) + int inListLength = inList.split(",").length; + if (inListLength == 1) { + // A single entry IN list is equivalent to an infinite range intersect; we use StatisticRange#INFINITE_TO_INFINITE_RANGE_INTERSECT_OVERLAP_HEURISTIC_FACTOR (0.5) + // as our filter factor, resulting in : non-null-inputRowCount * 0.5 = 50 + assertEquals(rowExpressionStatsEstimate.getOutputRowCount(), 50D); + } + else { + // Multiple values in the IN list - We sum up the estimates, but cap it to non-null-inputRowCount * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT = 80 in this case + assertEquals(rowExpressionStatsEstimate.getOutputRowCount(), 80D); + } + } + + @Test(dataProvider = "inList") + public void testNotInPredicateEstimateIsNeverZero(String inList) + { + RowExpression rowExpression = translator.translateAndOptimize(expression("mediumVarchar NOT IN (" + inList + ")"), standardTypes); + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session); + + assertNotEquals(rowExpressionStatsEstimate.getOutputRowCount(), 0D, 0.0001D); + } + @Test public void testInPredicateFilter() { @@ -588,7 +664,8 @@ public void testInPredicateFilter() // More values in range than distinct values assertExpression("z IN (DOUBLE '-1', 3.14e0, 0e0, 1e0, 2e0, 3e0, 4e0, 5e0, 6e0, 7e0, 8e0, DOUBLE '-2')") - .outputRowsCount(900.0) + // Range estimate is never the full-range, it's non-null count * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT + .outputRowsCount(720.0) .variableStats(new VariableReferenceExpression(Optional.empty(), "z", DOUBLE), variableStats -> variableStats.distinctValuesCount(5.0) .lowValue(-2.0) @@ -605,6 +682,34 @@ public void testInPredicateFilter() .nullsFraction(0.0)); } + @Test(dataProvider = "ndv1Expressions") + public void testNotEqualsOnVariablesWithNDV1(String expressionStr, double expectedOutputRowsCount) + { + Expression exp = expression(expressionStr); + + VariableReferenceExpression name = new VariableReferenceExpression(Optional.empty(), "name", MEDIUM_VARCHAR_TYPE); + TypeProvider customTypes = TypeProvider.fromVariables(ImmutableList.builder() + .add(name) + .build()); + + RowExpression rowExpression = translator.translateAndOptimize(exp, customTypes); + + VariableStatsEstimate nameStats = VariableStatsEstimate.builder() + .setNullsFraction(0D) + .setDistinctValuesCount(1D) + .build(); + + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(PlanNodeStatsEstimate.builder() + .addVariableStatistics(name, nameStats) + .setOutputRowCount(100D) + .build(), rowExpression, session); + + PlanNodeStatsAssertion.assertThat(rowExpressionStatsEstimate) + .outputRowsCount(expectedOutputRowsCount) + // Variable Stats remains unchanged + .variableStats(name, variableStats -> variableStats.distinctValuesCount(1D).nullsFraction(0D)); + } + protected PlanNodeStatsAssertion assertExpression(String expression) { return assertExpression(expression(expression)); diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java index 64bc0afd92f01..c837923ea9911 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java @@ -20,8 +20,8 @@ import java.util.concurrent.ThreadLocalRandom; import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; public class TestApproximateCountDistinctIpAddress extends AbstractTestApproximateCountDistinct diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java index e27fc7f2e57d1..349b2a190fe43 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java @@ -43,6 +43,8 @@ import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.P4HyperLogLogType.P4_HYPER_LOG_LOG; import static com.facebook.presto.common.type.QuantileDigestParametricType.QDIGEST; @@ -68,8 +70,6 @@ import static com.facebook.presto.type.ArrayParametricType.ARRAY; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.MapParametricType.MAP; import static com.facebook.presto.type.RowParametricType.ROW; import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java index 34a84c7c8a3bc..815a396bd27d7 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java @@ -43,6 +43,8 @@ import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.P4HyperLogLogType.P4_HYPER_LOG_LOG; import static com.facebook.presto.common.type.QuantileDigestParametricType.QDIGEST; @@ -68,8 +70,6 @@ import static com.facebook.presto.type.ArrayParametricType.ARRAY; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.MapParametricType.MAP; import static com.facebook.presto.type.RowParametricType.ROW; import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index e9b3e8641c885..dc8cf971df256 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -168,7 +168,8 @@ protected void assertInvalidFunction(String projection, StandardErrorCode errorC functionAssertions.assertInvalidFunction(projection, errorCode, messagePattern); } - protected void assertInvalidFunction(String projection, String messagePattern) + @Override + public void assertInvalidFunction(String projection, String messagePattern) { functionAssertions.assertInvalidFunction(projection, INVALID_FUNCTION_ARGUMENT, messagePattern); } @@ -208,7 +209,8 @@ protected void assertInvalidCast(String projection) functionAssertions.assertInvalidCast(projection); } - protected void assertInvalidCast(@Language("SQL") String projection, String message) + @Override + public void assertInvalidCast(@Language("SQL") String projection, String message) { functionAssertions.assertInvalidCast(projection, message); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java index 21f2b615887c9..61f80a92e5170 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java @@ -13,311 +13,10 @@ */ package com.facebook.presto.operator.scalar; -import com.facebook.presto.common.type.ArrayType; -import com.google.common.collect.ImmutableList; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import com.facebook.presto.tests.operator.scalar.AbstractTestIpPrefix; public class TestIpPrefixFunctions extends AbstractTestFunctions + implements AbstractTestIpPrefix { - @DataProvider(name = "public-ip-provider") - public Object[] publicIpProvider() - { - return new Object[] { - "6.7.8.9", - "157.240.200.99", - "8.8.8.8", - "128.1.2.8", - "2a03:2880:f031:12:face:b00c:0:2", - "2600:1406:6c00::173c:ad43", - "2607:f8b0:4007:818::2004" - }; - } - - @DataProvider(name = "private-ip-provider") - public Object[][] privateIpProvider() - { - return new Object[][] { - // The first and last IP address in each private range - {"0.0.0.0"}, {"0.255.255.255"}, // 0.0.0.0/8 RFC1122: "This host on this network" - {"10.0.0.0"}, {"10.255.255.255"}, // 10.0.0.0/8 RFC1918: Private-Use - {"100.64.0.0"}, {"100.127.255.255"}, // 100.64.0.0/10 RFC6598: Shared Address Space - {"127.0.0.0"}, {"127.255.255.255"}, // 127.0.0.0/8 RFC1122: Loopback - {"169.254.0.0"}, {"169.254.255.255"}, // 169.254.0.0/16 RFC3927: Link Local - {"172.16.0.0"}, {"172.31.255.255"}, // 172.16.0.0/12 RFC1918: Private-Use - {"192.0.0.0"}, {"192.0.0.255"}, // 192.0.0.0/24 RFC6890: IETF Protocol Assignments - {"192.0.2.0"}, {"192.0.2.255"}, // 192.0.2.0/24 RFC5737: Documentation (TEST-NET-1) - {"192.88.99.0"}, {"192.88.99.255"}, // 192.88.99.0/24 RFC3068: 6to4 Relay anycast - {"192.168.0.0"}, {"192.168.255.255"}, // 192.168.0.0/16 RFC1918: Private-Use - {"198.18.0.0"}, {"198.19.255.255"}, // 198.18.0.0/15 RFC2544: Benchmarking - {"198.51.100.0"}, {"198.51.100.255"}, // 198.51.100.0/24 RFC5737: Documentation (TEST-NET-2) - {"203.0.113.0"}, {"203.0.113.255"}, // 203.0.113.0/24 RFC5737: Documentation (TEST-NET-3) - {"240.0.0.0"}, {"255.255.255.255"}, // 240.0.0.0/4 RFC1112: Reserved - {"::"}, {"::"}, // ::/128 RFC4291: Unspecified address - {"::1"}, {"::1"}, // ::1/128 RFC4291: Loopback address - {"100::"}, {"100::ffff:ffff:ffff:ffff"}, // 100::/64 RFC6666: Discard-Only Address Block - {"64:ff9b:1::"}, {"64:ff9b:1:ffff:ffff:ffff:ffff:ffff"}, // 64:ff9b:1::/48 RFC8215: IPv4-IPv6 Translation - {"2001:2::"}, {"2001:2:0:ffff:ffff:ffff:ffff:ffff"}, // 2001:2::/48 RFC5180,RFC Errata 1752: Benchmarking - {"2001:db8::"}, {"2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001:db8::/32 RFC3849: Documentation - {"2001::"}, {"2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001::/23 RFC2928: IETF Protocol Assignments - {"5f00::"}, {"5f00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 5f00::/16 RFC-ietf-6man-sids-06: Segment Routing (SRv6) - {"fe80::"}, {"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fe80::/10 RFC4291: Link-Local Unicast - {"fc00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fc00::/7 RFC4193, RFC8190: Unique Local - // some IPs in the middle of ranges - {"10.1.2.3"}, - {"100.64.3.2"}, - {"192.168.55.99"}, - {"2001:0DB8:0000:0000:face:b00c:0000:0000"}, - {"0100:0000:0000:0000:ffff:ffff:0000:0000"} - }; - } - - @Test - public void testIpAddressIpPrefix() - { - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); - assertFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 0)", IPPREFIX, "::/0"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); - } - - @Test - public void testStringIpPrefix() - { - assertFunction("IP_PREFIX('1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX('1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); - assertFunction("IP_PREFIX('1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); - assertFunction("IP_PREFIX('::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX('64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); - assertFunction("IP_PREFIX('64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); - assertFunction("IP_PREFIX('64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); - assertFunction("IP_PREFIX('64:ff9b::17', 0)", IPPREFIX, "::/0"); - assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX('64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidFunction("IP_PREFIX('64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidCast("IP_PREFIX('localhost', 24)", "Cannot cast value to IPADDRESS: localhost"); - assertInvalidCast("IP_PREFIX('64::ff9b::10', 24)", "Cannot cast value to IPADDRESS: 64::ff9b::10"); - assertInvalidCast("IP_PREFIX('64:face:book::10', 24)", "Cannot cast value to IPADDRESS: 64:face:book::10"); - assertInvalidCast("IP_PREFIX('123.456.789.012', 24)", "Cannot cast value to IPADDRESS: 123.456.789.012"); - } - - @Test - public void testIpSubnetMin() - { - assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/24')", IPADDRESS, "1.2.3.0"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/32')", IPADDRESS, "1.2.3.4"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/127')", IPADDRESS, "64:ff9b::16"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/128')", IPADDRESS, "64:ff9b::17"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "::"); - } - - @Test - public void testIpSubnetMax() - { - assertFunction("IP_SUBNET_MAX(IPPREFIX '1.2.3.128/26')", IPADDRESS, "1.2.3.191"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '192.168.128.4/32')", IPADDRESS, "192.168.128.4"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '10.1.16.3/9')", IPADDRESS, "10.127.255.255"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/127')", IPADDRESS, "2001:db8::17"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/128')", IPADDRESS, "2001:db8::16"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::ffff:ffff:ffff:ffff"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/72')", IPADDRESS, "64:ff9b::ff:ffff:ffff:ffff"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); - } - - @Test - public void testIpSubnetRange() - { - assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.160/24')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.0", "1.2.3.255")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.128/31')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.128", "1.2.3.129")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/32')", new ArrayType(IPADDRESS), ImmutableList.of("10.1.6.46", "10.1.6.46")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/0')", new ArrayType(IPADDRESS), ImmutableList.of("0.0.0.0", "255.255.255.255")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/64')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::", "64:ff9b::ffff:ffff:ffff:ffff")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::52f4/120')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::5200", "64:ff9b::52ff")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/128')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::17", "64:ff9b::17")); - } - - @Test - public void testIsSubnetOf() - { - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.3.129')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.5.1')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/32', IPADDRESS '1.2.3.128')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/0', IPADDRESS '192.168.5.1')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ff9b::ffff:ff')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ffff::17')", BOOLEAN, false); - - assertFunction("IS_SUBNET_OF(IPPREFIX '192.168.3.131/26', IPPREFIX '192.168.3.144/30')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.5.1/30')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.3.128/26')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ff9b::ff:25/80')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ffff::17/64')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/37', IPPREFIX '2804:431:b000::/38')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/38', IPPREFIX '2804:431:b000::/37')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/22', IPPREFIX '170.0.52.0/24')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/24', IPPREFIX '170.0.52.0/22')", BOOLEAN, false); - } - - @Test - public void testIpv4PrefixCollapse() - { - // simple - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.1.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/23")); - - // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.1.0/24', IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.2.0/24', IPPREFIX '192.168.9.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/23", "192.168.2.0/24", "192.168.9.0/24")); - } - - @Test - public void testIpv6PrefixCollapse() - { - // simple - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090::/48', IPPREFIX '2620:10d:c091::/48'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2620:10d:c090::/47")); - - // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2804:13c:4d6:e200::/56', IPPREFIX '2804:13c:4d6:dd00::/56', IPPREFIX '2804:13c:4d6:dc00::/56', IPPREFIX '2804:13c:4d6:de00::/56'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2804:13c:4d6:dc00::/55", "2804:13c:4d6:de00::/56", "2804:13c:4d6:e200::/56")); - } - - @Test - public void testIpPrefixCollapseIpv4SingleIPs() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.1/32', IPPREFIX '192.168.33.1/32'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.1/32", "192.168.33.1/32")); - } - - @Test - public void testIpPrefixCollapseIpv6SingleIPs() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090:400::5:a869/128', IPPREFIX '2620:10d:c091:400::5:a869/128'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2620:10d:c090:400::5:a869/128", "2620:10d:c091:400::5:a869/128")); - } - - @Test - public void testIpPrefixCollapseSinglePrefixReturnsSamePrefix() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - } - - @Test - public void testIpPrefixCollapseOverlappingPrefixes() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.0.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.2.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.3.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.64.0/18', IPPREFIX '10.2.0.0/15', IPPREFIX '10.0.0.0/8', IPPREFIX '11.0.0.0/8', IPPREFIX '172.168.32.0/20', IPPREFIX '172.168.0.0/18'])", - new ArrayType(IPPREFIX), - ImmutableList.of("10.0.0.0/7", "172.168.0.0/18")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.0.0/8', IPPREFIX '10.0.0.0/7'])", - new ArrayType(IPPREFIX), - ImmutableList.of("10.0.0.0/7")); - } - - @Test - public void testIpPrefixCollapseEmptyArrayInput() - { - assertFunction("IP_PREFIX_COLLAPSE(CAST(ARRAY[] AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), ImmutableList.of()); - } - - @Test - public void testIpPrefixCollapseNullInput() - { - assertFunction("IP_PREFIX_COLLAPSE(CAST(NULL AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), null); - } - - @Test - public void testIpPrefixCollapseNoNullPrefixesError() - { - assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', CAST(NULL AS IPPREFIX)])", - "ip_prefix_collapse does not support null elements"); - } - - @Test - public void testIpPrefixCollapseMixedIpVersionError() - { - assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '2409:4043:251a:d200::/56'])", - "All IPPREFIX elements must be the same IP version."); - } - - @Test (dataProvider = "private-ip-provider") - public void testIsPrivateTrue(String ipAddress) - { - assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, true); - } - - @Test (dataProvider = "public-ip-provider") - public void testIsPrivateIpFalse(String ipAddress) - { - assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, false); - } - - @Test - public void testIsPrivateIpNull() - { - assertFunction("IS_PRIVATE_IP(NULL)", BOOLEAN, null); - } - - @Test - public void testIpPrefixSubnets() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 25)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/25", "192.168.1.128/25")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 26)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.0.0/26", "192.168.0.64/26", "192.168.0.128/26", "192.168.0.192/26")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2A03:2880:C000::/34', 37)", - new ArrayType(IPPREFIX), - ImmutableList.of("2a03:2880:c000::/37", "2a03:2880:c800::/37", "2a03:2880:d000::/37", "2a03:2880:d800::/37", "2a03:2880:e000::/37", "2a03:2880:e800::/37", "2a03:2880:f000::/37", "2a03:2880:f800::/37")); - } - - @Test - public void testIpPrefixSubnetsReturnSelf() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 24)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/24")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2804:431:b000::/38', 38)", new ArrayType(IPPREFIX), ImmutableList.of("2804:431:b000::/38")); - } - - @Test - public void testIpPrefixSubnetsNewPrefixLengthLongerReturnsEmpty() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 23)", new ArrayType(IPPREFIX), ImmutableList.of()); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 48)", new ArrayType(IPPREFIX), ImmutableList.of()); - } - - @Test - public void testIpPrefixSubnetsInvalidPrefixLengths() - { - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', -1)", "Invalid prefix length for IPv4: -1"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 33)", "Invalid prefix length for IPv4: 33"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', -1)", "Invalid prefix length for IPv6: -1"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 129)", "Invalid prefix length for IPv6: 129"); - } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java index 108ebb4da54e3..bf70110ea0eb5 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java @@ -450,6 +450,42 @@ public void testWithTableAlias() assertOptimizedQuery(baseQuerySqlWithTablePrefix, expectedRewrittenSql, originalViewSqlWithTablePrefix, BASE_TABLE_1, VIEW_1); } + @Test + public void testWithSchemaQualifiedTableName() + { + String schemaQualifiedTable = SESSION_SCHEMA + "." + BASE_TABLE_1; + + String originalViewSql = format("SELECT a, b FROM %s", BASE_TABLE_1); + String baseQuerySql = format("SELECT a, b FROM %s", schemaQualifiedTable); + String expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b FROM %s", schemaQualifiedTable); + baseQuerySql = format("SELECT a, b FROM %s", BASE_TABLE_1); + expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b FROM %s", schemaQualifiedTable); + baseQuerySql = format("SELECT a, b FROM %s", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b, c FROM %s", BASE_TABLE_1); + baseQuerySql = format("SELECT a, b FROM %s WHERE c > 10", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT a, b FROM %s WHERE c > 10", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT SUM(a) as sum_a, b FROM %s GROUP BY b", BASE_TABLE_1); + baseQuerySql = format("SELECT SUM(a), b FROM %s GROUP BY b", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT SUM(sum_a), b FROM %s GROUP BY b", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + } + @Test public void testAggregationWithTableAlias() { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java index d19e911fc2f60..8ba19fb23f05f 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java @@ -31,11 +31,11 @@ import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; public class TestCheckUnsupportedPrestissimoTypes extends BasePlanTest diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java index 7c1d54dbfad49..b83540d2c05c4 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java @@ -25,9 +25,9 @@ import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.io.BaseEncoding.base16; public class TestIpAddressOperators diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java index cacd4b3ad252c..39a4ec375d253 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java @@ -20,7 +20,7 @@ import io.airlift.slice.Slices; import org.testng.annotations.Test; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static org.testng.Assert.assertEquals; public class TestIpAddressType diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java index cbd9dff770e28..77f3280787539 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java @@ -24,9 +24,9 @@ import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static java.lang.System.arraycopy; public class TestIpPrefixOperators diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java index 50b5eb672708a..7a3a6d94337d7 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java @@ -20,7 +20,7 @@ import io.airlift.slice.Slices; import org.testng.annotations.Test; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.google.common.base.Preconditions.checkState; import static java.lang.System.arraycopy; import static org.testng.Assert.assertEquals; diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java new file mode 100644 index 0000000000000..b3ade52608e85 --- /dev/null +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java @@ -0,0 +1,324 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.operator.scalar; + +import com.facebook.presto.common.type.ArrayType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; + +public interface AbstractTestIpPrefix + extends TestFunctions +{ + @DataProvider(name = "public-ip-provider") + default Object[] publicIpProvider() + { + return new Object[] { + "6.7.8.9", + "157.240.200.99", + "8.8.8.8", + "128.1.2.8", + "2a03:2880:f031:12:face:b00c:0:2", + "2600:1406:6c00::173c:ad43", + "2607:f8b0:4007:818::2004" + }; + } + + @DataProvider(name = "private-ip-provider") + default Object[][] privateIpProvider() + { + return new Object[][] { + // The first and last IP address in each private range + {"0.0.0.0"}, {"0.255.255.255"}, // 0.0.0.0/8 RFC1122: "This host on this network" + {"10.0.0.0"}, {"10.255.255.255"}, // 10.0.0.0/8 RFC1918: Private-Use + {"100.64.0.0"}, {"100.127.255.255"}, // 100.64.0.0/10 RFC6598: Shared Address Space + {"127.0.0.0"}, {"127.255.255.255"}, // 127.0.0.0/8 RFC1122: Loopback + {"169.254.0.0"}, {"169.254.255.255"}, // 169.254.0.0/16 RFC3927: Link Local + {"172.16.0.0"}, {"172.31.255.255"}, // 172.16.0.0/12 RFC1918: Private-Use + {"192.0.0.0"}, {"192.0.0.255"}, // 192.0.0.0/24 RFC6890: IETF Protocol Assignments + {"192.0.2.0"}, {"192.0.2.255"}, // 192.0.2.0/24 RFC5737: Documentation (TEST-NET-1) + {"192.88.99.0"}, {"192.88.99.255"}, // 192.88.99.0/24 RFC3068: 6to4 Relay anycast + {"192.168.0.0"}, {"192.168.255.255"}, // 192.168.0.0/16 RFC1918: Private-Use + {"198.18.0.0"}, {"198.19.255.255"}, // 198.18.0.0/15 RFC2544: Benchmarking + {"198.51.100.0"}, {"198.51.100.255"}, // 198.51.100.0/24 RFC5737: Documentation (TEST-NET-2) + {"203.0.113.0"}, {"203.0.113.255"}, // 203.0.113.0/24 RFC5737: Documentation (TEST-NET-3) + {"240.0.0.0"}, {"255.255.255.255"}, // 240.0.0.0/4 RFC1112: Reserved + {"::"}, {"::"}, // ::/128 RFC4291: Unspecified address + {"::1"}, {"::1"}, // ::1/128 RFC4291: Loopback address + {"100::"}, {"100::ffff:ffff:ffff:ffff"}, // 100::/64 RFC6666: Discard-Only Address Block + {"64:ff9b:1::"}, {"64:ff9b:1:ffff:ffff:ffff:ffff:ffff"}, // 64:ff9b:1::/48 RFC8215: IPv4-IPv6 Translation + {"2001:2::"}, {"2001:2:0:ffff:ffff:ffff:ffff:ffff"}, // 2001:2::/48 RFC5180,RFC Errata 1752: Benchmarking + {"2001:db8::"}, {"2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001:db8::/32 RFC3849: Documentation + {"2001::"}, {"2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001::/23 RFC2928: IETF Protocol Assignments + {"5f00::"}, {"5f00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 5f00::/16 RFC-ietf-6man-sids-06: Segment Routing (SRv6) + {"fe80::"}, {"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fe80::/10 RFC4291: Link-Local Unicast + {"fc00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fc00::/7 RFC4193, RFC8190: Unique Local + // some IPs in the middle of ranges + {"10.1.2.3"}, + {"100.64.3.2"}, + {"192.168.55.99"}, + {"2001:0DB8:0000:0000:face:b00c:0000:0000"}, + {"0100:0000:0000:0000:ffff:ffff:0000:0000"} + }; + } + + @Test + default void testIpAddressIpPrefix() + { + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + } + + @Test + default void testStringIpPrefix() + { + assertFunction("IP_PREFIX('1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX('1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX('::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX('64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX('64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX('64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidCast("IP_PREFIX('localhost', 24)", "Cannot cast value to IPADDRESS: localhost"); + assertInvalidCast("IP_PREFIX('64::ff9b::10', 24)", "Cannot cast value to IPADDRESS: 64::ff9b::10"); + assertInvalidCast("IP_PREFIX('64:face:book::10', 24)", "Cannot cast value to IPADDRESS: 64:face:book::10"); + assertInvalidCast("IP_PREFIX('123.456.789.012', 24)", "Cannot cast value to IPADDRESS: 123.456.789.012"); + } + + @Test + default void testIpSubnetMin() + { + assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/24')", IPADDRESS, "1.2.3.0"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/32')", IPADDRESS, "1.2.3.4"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/127')", IPADDRESS, "64:ff9b::16"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/128')", IPADDRESS, "64:ff9b::17"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "::"); + } + + @Test + default void testIpSubnetMax() + { + assertFunction("IP_SUBNET_MAX(IPPREFIX '1.2.3.128/26')", IPADDRESS, "1.2.3.191"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '192.168.128.4/32')", IPADDRESS, "192.168.128.4"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '10.1.16.3/9')", IPADDRESS, "10.127.255.255"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/127')", IPADDRESS, "2001:db8::17"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/128')", IPADDRESS, "2001:db8::16"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::ffff:ffff:ffff:ffff"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/72')", IPADDRESS, "64:ff9b::ff:ffff:ffff:ffff"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + } + + @Test + default void testIpSubnetRange() + { + assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.160/24')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.0", "1.2.3.255")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.128/31')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.128", "1.2.3.129")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/32')", new ArrayType(IPADDRESS), ImmutableList.of("10.1.6.46", "10.1.6.46")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/0')", new ArrayType(IPADDRESS), ImmutableList.of("0.0.0.0", "255.255.255.255")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/64')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::", "64:ff9b::ffff:ffff:ffff:ffff")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::52f4/120')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::5200", "64:ff9b::52ff")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/128')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::17", "64:ff9b::17")); + } + + @Test + default void testIsSubnetOf() + { + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.3.129')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.5.1')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/32', IPADDRESS '1.2.3.128')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/0', IPADDRESS '192.168.5.1')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ff9b::ffff:ff')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ffff::17')", BOOLEAN, false); + + assertFunction("IS_SUBNET_OF(IPPREFIX '192.168.3.131/26', IPPREFIX '192.168.3.144/30')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.5.1/30')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.3.128/26')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ff9b::ff:25/80')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ffff::17/64')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/37', IPPREFIX '2804:431:b000::/38')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/38', IPPREFIX '2804:431:b000::/37')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/22', IPPREFIX '170.0.52.0/24')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/24', IPPREFIX '170.0.52.0/22')", BOOLEAN, false); + } + + @Test + default void testIpv4PrefixCollapse() + { + // simple + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.1.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/23")); + + // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.1.0/24', IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.2.0/24', IPPREFIX '192.168.9.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/23", "192.168.2.0/24", "192.168.9.0/24")); + } + + @Test + default void testIpv6PrefixCollapse() + { + // simple + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090::/48', IPPREFIX '2620:10d:c091::/48'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2620:10d:c090::/47")); + + // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2804:13c:4d6:e200::/56', IPPREFIX '2804:13c:4d6:dd00::/56', IPPREFIX '2804:13c:4d6:dc00::/56', IPPREFIX '2804:13c:4d6:de00::/56'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2804:13c:4d6:dc00::/55", "2804:13c:4d6:de00::/56", "2804:13c:4d6:e200::/56")); + } + + @Test + default void testIpPrefixCollapseIpv4SingleIPs() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.1/32', IPPREFIX '192.168.33.1/32'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.1/32", "192.168.33.1/32")); + } + + @Test + default void testIpPrefixCollapseIpv6SingleIPs() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090:400::5:a869/128', IPPREFIX '2620:10d:c091:400::5:a869/128'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2620:10d:c090:400::5:a869/128", "2620:10d:c091:400::5:a869/128")); + } + + @Test + default void testIpPrefixCollapseSinglePrefixReturnsSamePrefix() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + } + + @Test + default void testIpPrefixCollapseOverlappingPrefixes() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.0.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.2.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.3.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.64.0/18', IPPREFIX '10.2.0.0/15', IPPREFIX '10.0.0.0/8', IPPREFIX '11.0.0.0/8', IPPREFIX '172.168.32.0/20', IPPREFIX '172.168.0.0/18'])", + new ArrayType(IPPREFIX), + ImmutableList.of("10.0.0.0/7", "172.168.0.0/18")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.0.0/8', IPPREFIX '10.0.0.0/7'])", + new ArrayType(IPPREFIX), + ImmutableList.of("10.0.0.0/7")); + } + + @Test + default void testIpPrefixCollapseEmptyArrayInput() + { + assertFunction("IP_PREFIX_COLLAPSE(CAST(ARRAY[] AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), ImmutableList.of()); + } + + @Test + default void testIpPrefixCollapseNullInput() + { + assertFunction("IP_PREFIX_COLLAPSE(CAST(NULL AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), null); + } + + @Test + default void testIpPrefixCollapseNoNullPrefixesError() + { + assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', CAST(NULL AS IPPREFIX)])", + "ip_prefix_collapse does not support null elements"); + } + + @Test + default void testIpPrefixCollapseMixedIpVersionError() + { + assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '2409:4043:251a:d200::/56'])", + "All IPPREFIX elements must be the same IP version."); + } + + @Test (dataProvider = "private-ip-provider") + default void testIsPrivateTrue(String ipAddress) + { + assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, true); + } + + @Test (dataProvider = "public-ip-provider") + default void testIsPrivateIpFalse(String ipAddress) + { + assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, false); + } + + @Test + default void testIsPrivateIpNull() + { + assertFunction("IS_PRIVATE_IP(NULL)", BOOLEAN, null); + } + + @Test + default void testIpPrefixSubnets() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 25)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/25", "192.168.1.128/25")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 26)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.0.0/26", "192.168.0.64/26", "192.168.0.128/26", "192.168.0.192/26")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2A03:2880:C000::/34', 37)", + new ArrayType(IPPREFIX), + ImmutableList.of("2a03:2880:c000::/37", "2a03:2880:c800::/37", "2a03:2880:d000::/37", "2a03:2880:d800::/37", "2a03:2880:e000::/37", "2a03:2880:e800::/37", "2a03:2880:f000::/37", "2a03:2880:f800::/37")); + } + + @Test + default void testIpPrefixSubnetsReturnSelf() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 24)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/24")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2804:431:b000::/38', 38)", new ArrayType(IPPREFIX), ImmutableList.of("2804:431:b000::/38")); + } + + @Test + default void testIpPrefixSubnetsNewPrefixLengthLongerReturnsEmpty() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 23)", new ArrayType(IPPREFIX), ImmutableList.of()); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 48)", new ArrayType(IPPREFIX), ImmutableList.of()); + } + + @Test + default void testIpPrefixSubnetsInvalidPrefixLengths() + { + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', -1)", "Invalid prefix length for IPv4: -1"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 33)", "Invalid prefix length for IPv4: 33"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', -1)", "Invalid prefix length for IPv6: -1"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 129)", "Invalid prefix length for IPv6: 129"); + } +} diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java index e72644201ef3b..01a008b62296b 100644 --- a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java @@ -27,4 +27,16 @@ public interface TestFunctions * Asserts that the projection is not supported and that it fails with the expected error message. */ void assertNotSupported(String projection, String message); + + /** + * Asserts that the projection contains an invalid function call and fails + * with the specified functional error message. + */ + void assertInvalidFunction(String projection, String message); + + /** + * Asserts that the projection contains an invalid type conversion (cast) + * and fails with the expected casting error message. + */ + void assertInvalidCast(String projection, String message); } diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 9c06fa12d4bdd..ba0e9f01ed016 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -1073,7 +1073,8 @@ void PrestoServer::initializeVeloxMemory() { velox::cache::AsyncDataCache::Options cacheOptions{ systemConfig->asyncCacheMaxSsdWriteRatio(), systemConfig->asyncCacheSsdSavableRatio(), - systemConfig->asyncCacheMinSsdSavableBytes()}; + systemConfig->asyncCacheMinSsdSavableBytes(), + systemConfig->asyncCacheNumShards()}; cache_ = velox::cache::AsyncDataCache::create( velox::memory::memoryManager()->allocator(), std::move(ssd), diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index 89f14085da3e2..e44937470a186 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -209,6 +209,7 @@ SystemConfig::SystemConfig() { NUM_PROP(kAsyncCacheMaxSsdWriteRatio, 0.7), NUM_PROP(kAsyncCacheSsdSavableRatio, 0.125), NUM_PROP(kAsyncCacheMinSsdSavableBytes, 1 << 24 /*16MB*/), + NUM_PROP(kAsyncCacheNumShards, 4), STR_PROP(kAsyncCachePersistenceInterval, "0s"), BOOL_PROP(kAsyncCacheSsdDisableFileCow, false), BOOL_PROP(kSsdCacheChecksumEnabled, false), @@ -695,6 +696,10 @@ int32_t SystemConfig::asyncCacheMinSsdSavableBytes() const { return optionalProperty(kAsyncCacheMinSsdSavableBytes).value(); } +int32_t SystemConfig::asyncCacheNumShards() const { + return optionalProperty(kAsyncCacheNumShards).value(); +} + std::chrono::duration SystemConfig::asyncCachePersistenceInterval() const { return velox::config::toDuration( diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index 2c44c3e845427..b92fa33c255b5 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -429,6 +429,12 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kAsyncCacheMinSsdSavableBytes{ "async-cache-min-ssd-savable-bytes"}; + /// The number of shards for the async data cache. The cache is divided into + /// shards to decrease contention on the mutex for the key to entry mapping + /// and other housekeeping. Must be a power of 2. + static constexpr std::string_view kAsyncCacheNumShards{ + "async-cache-num-shards"}; + /// The interval for persisting in-memory cache to SSD. Setting this config /// to a non-zero value will activate periodic cache persistence. static constexpr std::string_view kAsyncCachePersistenceInterval{ @@ -1066,6 +1072,8 @@ class SystemConfig : public ConfigBase { int32_t asyncCacheMinSsdSavableBytes() const; + int32_t asyncCacheNumShards() const; + std::chrono::duration asyncCachePersistenceInterval() const; bool asyncCacheSsdDisableFileCow() const; diff --git a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp index 76096dad1145a..694ce3d14b251 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp @@ -230,6 +230,17 @@ TEST_F(ConfigTest, optionalSystemConfigsWithDefault) { ASSERT_EQ(config.maxDriversPerTask(), 1024); } +TEST_F(ConfigTest, asyncCacheNumShards) { + SystemConfig config; + init(config, {}); + // Test default value is 4 + ASSERT_EQ(config.asyncCacheNumShards(), 4); + + // Test custom value + init(config, {{std::string(SystemConfig::kAsyncCacheNumShards), "8"}}); + ASSERT_EQ(config.asyncCacheNumShards(), 8); +} + TEST_F(ConfigTest, remoteFunctionServer) { SystemConfig config; init(config, {}); diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java index 6c99067f3f399..3952d8e82ef1a 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java @@ -80,6 +80,34 @@ public void assertNotSupported(String projection, @Language("RegExp") String mes } } + @Override + public void assertInvalidFunction(String projection, @Language("RegExp") String message) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + try { + computeActual(rewritten); + fail("expected exception"); + } + catch (RuntimeException ex) { + assertExceptionMessage(rewritten, ex, Pattern.quote(message), true, false); + } + } + + @Override + public void assertInvalidCast(String projection, @Language("RegExp") String message) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + try { + computeActual(rewritten); + fail("expected exception"); + } + catch (RuntimeException ex) { + assertExceptionMessage(rewritten, ex, message, true, false); + } + } + /** * Rewrite SQL of the form 'select cast(arg as type)' to 'select cast(a as type) from (values (arg)) t(a)', and * SQL of the form 'select function(arg1, arg2, ...)' to diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java new file mode 100644 index 0000000000000..ea181eee17544 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests.operator.scalar; + +import com.facebook.presto.tests.operator.scalar.AbstractTestIpPrefix; + +public class TestIpPrefixFunctions + extends AbstractTestNativeFunctions + implements AbstractTestIpPrefix +{ +} diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c703a051062d4..a84c67a450657 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -22,6 +22,12 @@ com.facebook.presto presto-common + + + com.google.guava + guava + + @@ -73,6 +79,12 @@ presto-common test-jar test + + + com.google.guava + guava + + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java index 904c61831f0fa..db2cafbda26d4 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java @@ -70,6 +70,8 @@ import static com.facebook.presto.common.type.DateType.DATE; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.RealType.REAL; import static com.facebook.presto.common.type.SmallintType.SMALLINT; @@ -82,7 +84,6 @@ import static com.facebook.presto.testing.MaterializedResult.DEFAULT_PRECISION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.transform; @@ -246,6 +247,9 @@ else if (INTERVAL_YEAR_MONTH.equals(type)) { else if (IPADDRESS.equals(type)) { return value; } + else if (IPPREFIX.equals(type)) { + return value; + } else if (type instanceof ArrayType) { return ((List) value).stream() .map(element -> convertToRowValue(((ArrayType) type).getElementType(), element))