diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 8c89b03100e6..615d68ed8ce4 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -108,6 +108,22 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty ENABLE_REX_DIGEST_NORMALIZE = booleanProperty("calcite.enable.rexnode.digest.normalize", true); + /** Maximum number of candidate expressions constructed by one expression + * lineage expansion. Candidates are counted before deduplication. If the + * Cartesian product of column origins exceeds this budget, the metadata + * provider returns unknown rather than a partial result. + * + *

The default is 10,000. A value of -1 disables the limit; 0 disables + * expansions that reference input columns. Expressions without input + * references do not require expansion and are unaffected. Values less than + * -1 are invalid and use the default. + * + *

This is a per-expansion work limit, not a query-wide memory limit. It + * also applies when all-predicates metadata uses the same expansion helper. + */ + public static final CalciteSystemProperty EXPRESSION_LINEAGE_MAX_EXPANSIONS = + intProperty("calcite.metadata.expression.lineage.max.expansions", 10_000, v -> v >= -1); + /** * Whether to follow the SQL standard strictly. */ diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index 9c571dfddcd3..32d4dd464220 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -553,7 +553,8 @@ public interface ExpressionLineage extends Metadata { * * @return set of expressions with lineage resolved, or null if this information * cannot be determined (e.g. origin of an expression is an aggregation - * in an {@link org.apache.calcite.rel.core.Aggregate} operator) + * in an {@link org.apache.calcite.rel.core.Aggregate} operator), or if the + * candidate expansion exceeds the provider's work budget */ @Nullable Set getExpressionLineage(RexNode expression); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java index 7ff467d1a7d6..7890582c049b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel.metadata; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; @@ -66,6 +67,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkArgument; + import static java.util.Objects.requireNonNull; /** @@ -80,7 +83,9 @@ * TableScan operator is identified uniquely by a {@link RelTableRef} containing * its qualified name and an identifier. * - *

If the lineage cannot be inferred, we return null. + *

If the lineage cannot be inferred, or expanding its candidate expressions + * exceeds {@link CalciteSystemProperty#EXPRESSION_LINEAGE_MAX_EXPANSIONS}, + * we return null. */ public class RelMdExpressionLineage implements MetadataHandler { @@ -494,10 +499,20 @@ protected RelMdExpressionLineage() {} * @param rexBuilder rexBuilder * @param expr expression * @param mapping mapping - * @return set of resulting expressions equivalent to the input expression + * @return set of resulting expressions equivalent to the input expression, + * or null if expansion is unsupported or exceeds the configured budget */ protected static @Nullable Set createAllPossibleExpressions(RexBuilder rexBuilder, RexNode expr, Map> mapping) { + return createAllPossibleExpressions(rexBuilder, expr, mapping, + CalciteSystemProperty.EXPRESSION_LINEAGE_MAX_EXPANSIONS.value()); + } + + /** Expands with an explicit per-call budget; -1 means unlimited. + * The budget counts candidate construction attempts, not distinct results. */ + static @Nullable Set createAllPossibleExpressions(RexBuilder rexBuilder, + RexNode expr, Map> mapping, int maxExpansions) { + checkArgument(maxExpansions >= -1, "maxExpansions must be at least -1"); // Extract input fields referenced by expression final ImmutableBitSet predFieldsUsed = extractInputRefs(expr); @@ -506,6 +521,24 @@ protected RelMdExpressionLineage() {} return ImmutableSet.of(expr); } + if (maxExpansions >= 0) { + // Estimate the number of candidates before copying any RexNodes. + int remaining = maxExpansions; + for (Map.Entry> entry : mapping.entrySet()) { + if (predFieldsUsed.get(entry.getKey().getIndex())) { + int count = entry.getValue().size(); + if (count > remaining) { + return null; + } + if (count > 0) { + // Divide the remaining budget instead of multiplying the counts: + // the product can overflow even when each count fits in an int. + remaining /= count; + } + } + } + } + try { return createAllPossibleExpressions(rexBuilder, expr, predFieldsUsed, mapping, new HashMap<>()); diff --git a/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java new file mode 100644 index 000000000000..58f6b5baf4d1 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.google.common.base.Preconditions.checkArgument; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** Tests for bounded expression-lineage expansion. */ +class RelMdExpressionLineageTest { + @ParameterizedTest + @CsvSource({"2, 2, false", "10, 4, true", "11, 4, true", "11, 4, false"}) + void testUnionLineage(int branches, int columns, boolean window) throws Exception { + RelNode rel = ExpressionLineageProgram.rel(branches, columns, window); + RexNode computed = RexInputRef.of(columns, rel.getRowType().getFieldList()); + Set lineage = rel.getCluster().getMetadataQuery().getExpressionLineage(rel, computed); + int candidates = (int) Math.pow(branches, columns); + int budget = CalciteSystemProperty.EXPRESSION_LINEAGE_MAX_EXPANSIONS.value(); + if (budget >= 0 && candidates > budget) { + // Metadata was not computed. + assertNull(lineage); + } else { + assertNotNull(lineage); + assertEquals(candidates, lineage.size()); + } + } + + @Test void testOverflowDoesNotStartExpansion() { + RexBuilder builder = new RexBuilder(new JavaTypeFactoryImpl()); + RelDataType type = builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + RexInputRef a = builder.makeInputRef(type, 0); + RexInputRef b = builder.makeInputRef(type, 1); + Set origins = new AbstractSet() { + @Override public int size() { + return 65_536; + } + + @Override public Iterator iterator() { + throw new AssertionError("Over-budget origins must not be enumerated"); + } + }; + Map> mapping = new LinkedHashMap<>(); + mapping.put(a, origins); + mapping.put(b, origins); + // 65,536 squared wraps to zero as an int. + assertNull( + RelMdExpressionLineage.createAllPossibleExpressions(builder, + builder.makeCall(SqlStdOperatorTable.MINUS, a, b), mapping, Integer.MAX_VALUE)); + } + + /** Generates UNION ALL plans for expression-lineage tests. */ + private static final class ExpressionLineageProgram { + private ExpressionLineageProgram() { + } + + /** Generates a query whose final column references every column of a UNION ALL. + * For example, {@code sql(2, 2, true)} generates: + *

{@code
+     * SELECT c0, c1, ROW_NUMBER() OVER (PARTITION BY c0 ORDER BY c1) AS computed_value
+     * FROM (SELECT c0, c1 FROM events UNION ALL SELECT c0, c1 FROM events) u
+     * }
*/ + static String sql(int branches, int columns, boolean window) { + checkArgument(branches >= 1, "branches must be positive"); + checkArgument(columns >= 1, "columns must be positive"); + String fields = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")); + String expression; + if (window) { + String partition = columns == 1 ? "" : "partition by " + + IntStream.range(0, columns - 1).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")) + " "; + expression = "row_number() over (" + partition + "order by c" + (columns - 1) + ")"; + } else { + expression = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(" + ")); + } + String inputs = IntStream.range(0, branches) + .mapToObj(i -> "select " + fields + " from events") + .collect(Collectors.joining(" union all ")); + return "select " + fields + ", " + expression + " as computed_value from (" + inputs + ") u"; + } + + /** Returns the relational plan for {@link #sql(int, int, boolean)}. The lineage of + * its computed column has up to {@code branches ^ columns} candidate expressions. */ + static RelNode rel(int branches, int columns, boolean window) throws Exception { + String sql = sql(branches, columns, window); + SchemaPlus schema = Frameworks.createRootSchema(true); + schema.add("EVENTS", new AbstractTable() { + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (int i = 0; i < columns; i++) { + builder.add("C" + i, SqlTypeName.INTEGER); + } + return builder.build(); + } + }); + try (Planner planner = Frameworks.getPlanner(Frameworks.newConfigBuilder() + .defaultSchema(schema).build())) { + SqlNode validated = planner.validate(planner.parse(sql)); + return planner.rel(validated).project(); + } + } + } +}