From 311d63f767735cd512533220e7e20f70145dfa14 Mon Sep 17 00:00:00 2001 From: ShengHuang Date: Sat, 7 Nov 2026 02:35:33 +0800 Subject: [PATCH 1/2] [CALCITE-7924] RelMdExpressionLineage Cartesian expansion over UNION ALL causes excessive memory use and GC Bound Cartesian candidate construction before copying expressions, returning unknown metadata rather than a partial result when the configurable per-expansion budget is exceeded. Preserve within-budget lineage and add a parameterized SQL generator with Project, Calc, all-predicates, and budget-boundary regression coverage. --- .../calcite/config/CalciteSystemProperty.java | 16 ++ .../calcite/rel/metadata/BuiltInMetadata.java | 3 +- .../rel/metadata/RelMdExpressionLineage.java | 37 ++- .../metadata/ExpressionLineageProgram.java | 83 ++++++ .../metadata/RelMdExpressionLineageTest.java | 243 ++++++++++++++++++ 5 files changed, 379 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java create mode 100644 core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 8c89b03100e6..615d68ed8ce4 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -108,6 +108,22 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty ENABLE_REX_DIGEST_NORMALIZE = booleanProperty("calcite.enable.rexnode.digest.normalize", true); + /** Maximum number of candidate expressions constructed by one expression + * lineage expansion. Candidates are counted before deduplication. If the + * Cartesian product of column origins exceeds this budget, the metadata + * provider returns unknown rather than a partial result. + * + *

The default is 10,000. A value of -1 disables the limit; 0 disables + * expansions that reference input columns. Expressions without input + * references do not require expansion and are unaffected. Values less than + * -1 are invalid and use the default. + * + *

This is a per-expansion work limit, not a query-wide memory limit. It + * also applies when all-predicates metadata uses the same expansion helper. + */ + public static final CalciteSystemProperty EXPRESSION_LINEAGE_MAX_EXPANSIONS = + intProperty("calcite.metadata.expression.lineage.max.expansions", 10_000, v -> v >= -1); + /** * Whether to follow the SQL standard strictly. */ diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index 9c571dfddcd3..32d4dd464220 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -553,7 +553,8 @@ public interface ExpressionLineage extends Metadata { * * @return set of expressions with lineage resolved, or null if this information * cannot be determined (e.g. origin of an expression is an aggregation - * in an {@link org.apache.calcite.rel.core.Aggregate} operator) + * in an {@link org.apache.calcite.rel.core.Aggregate} operator), or if the + * candidate expansion exceeds the provider's work budget */ @Nullable Set getExpressionLineage(RexNode expression); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java index 7ff467d1a7d6..d3ade7696c93 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.rel.metadata; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; @@ -66,6 +67,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkArgument; + import static java.util.Objects.requireNonNull; /** @@ -80,7 +83,9 @@ * TableScan operator is identified uniquely by a {@link RelTableRef} containing * its qualified name and an identifier. * - *

If the lineage cannot be inferred, we return null. + *

If the lineage cannot be inferred, or expanding its candidate expressions + * exceeds {@link CalciteSystemProperty#EXPRESSION_LINEAGE_MAX_EXPANSIONS}, + * we return null. */ public class RelMdExpressionLineage implements MetadataHandler { @@ -494,10 +499,20 @@ protected RelMdExpressionLineage() {} * @param rexBuilder rexBuilder * @param expr expression * @param mapping mapping - * @return set of resulting expressions equivalent to the input expression + * @return set of resulting expressions equivalent to the input expression, + * or null if expansion is unsupported or exceeds the configured budget */ protected static @Nullable Set createAllPossibleExpressions(RexBuilder rexBuilder, RexNode expr, Map> mapping) { + return createAllPossibleExpressions(rexBuilder, expr, mapping, + CalciteSystemProperty.EXPRESSION_LINEAGE_MAX_EXPANSIONS.value()); + } + + /** Expands with an explicit per-call budget; -1 means unlimited. + * The budget counts candidate construction attempts, not distinct results. */ + static @Nullable Set createAllPossibleExpressions(RexBuilder rexBuilder, + RexNode expr, Map> mapping, int maxExpansions) { + checkArgument(maxExpansions >= -1, "maxExpansions must be at least -1"); // Extract input fields referenced by expression final ImmutableBitSet predFieldsUsed = extractInputRefs(expr); @@ -506,6 +521,24 @@ protected RelMdExpressionLineage() {} return ImmutableSet.of(expr); } + if (maxExpansions >= 0) { + // Every referenced column adds a dimension to the Cartesian product. + // Count before copying RexNodes, even if equal candidates would later + // collapse in a Set. Division avoids overflow for very large products. + int remaining = maxExpansions; + for (Map.Entry> entry : mapping.entrySet()) { + if (predFieldsUsed.get(entry.getKey().getIndex())) { + int count = entry.getValue().size(); + if (count > remaining) { + return null; + } + if (count > 0) { + remaining /= count; + } + } + } + } + try { return createAllPossibleExpressions(rexBuilder, expr, predFieldsUsed, mapping, new HashMap<>()); diff --git a/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java b/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java new file mode 100644 index 000000000000..2aca64b2d5c3 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; + +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.google.common.base.Preconditions.checkArgument; + +/** Generates storage-free SQL programs whose expression lineage has + * {@code branches ^ columns} candidate substitutions. Shared by regression + * tests and standalone measurements. */ +final class ExpressionLineageProgram { + private ExpressionLineageProgram() { + } + + /** Generates a SQL program with passthrough columns and a final computed + * column, either an arithmetic expression or a window expression. */ + static String sql(int branches, int columns, boolean window) { + checkArgument(branches >= 1, "branches must be positive"); + checkArgument(columns >= 1, "columns must be positive"); + String fields = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")); + String expression; + if (window) { + String partition = columns == 1 ? "" : "partition by " + + IntStream.range(0, columns - 1).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")) + " "; + expression = "row_number() over (" + partition + "order by c" + (columns - 1) + ")"; + } else { + expression = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(" + ")); + } + String inputs = IntStream.range(0, branches) + .mapToObj(i -> "select " + fields + " from events") + .collect(Collectors.joining(" union all ")); + return "select " + fields + ", " + expression + " as computed_value from (" + inputs + ") u"; + } + + /** Parses and validates the generated SQL using only Calcite. */ + static RelNode rel(int branches, int columns, boolean window) throws Exception { + String sql = sql(branches, columns, window); + SchemaPlus schema = Frameworks.createRootSchema(true); + schema.add("EVENTS", new AbstractTable() { + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (int i = 0; i < columns; i++) { + builder.add("C" + i, SqlTypeName.INTEGER); + } + return builder.build(); + } + }); + try (Planner planner = Frameworks.getPlanner(Frameworks.newConfigBuilder() + .defaultSchema(schema).build())) { + SqlNode validated = planner.validate(planner.parse(sql)); + return planner.rel(validated).project(); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java new file mode 100644 index 000000000000..671f7fae8338 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptPredicateList; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Calc; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; + +import com.google.common.collect.ImmutableSet; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.math.BigDecimal; +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Tests for bounded expression-lineage expansion. */ +class RelMdExpressionLineageTest { + private final RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl()); + + @ParameterizedTest + @CsvSource({"1, 1", "2, 2", "2, 4", "4, 4", "8, 4", "16, 4", "90, 4", "2, 16"}) + void testGeneratedPrograms(int branches, int columns) throws Exception { + for (boolean window : new boolean[] {false, true}) { + for (boolean calc : new boolean[] {false, true}) { + RelNode rel = ExpressionLineageProgram.rel(branches, columns, window); + if (calc) { + HepPlanner planner = new HepPlanner(new HepProgramBuilder() + .addRuleInstance(CoreRules.PROJECT_TO_CALC).build()); + planner.setRoot(rel); + rel = planner.findBestExp(); + assertTrue(rel instanceof Calc, "Expected the Calc metadata path"); + } + RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); + RexNode result = RexInputRef.of(columns, rel.getRowType().getFieldList()); + Set lineage = mq.getExpressionLineage(rel, result); + if (Math.pow(branches, columns) > 10_000) { + // No truncated subset may escape as complete metadata. + assertNull(lineage); + } else { + assertNotNull(lineage); + assertEquals((int) Math.pow(branches, columns), lineage.size()); + } + // An expensive result must not poison other columns or later queries. + Set passthrough = mq.getExpressionLineage(rel, + RexInputRef.of(0, rel.getRowType().getFieldList())); + assertNotNull(passthrough); + assertEquals(branches, passthrough.size()); + assertEquals(lineage, mq.getExpressionLineage(rel, result)); + rel.getCluster().invalidateMetadataQuery(); + assertEquals(lineage, rel.getCluster().getMetadataQuery() + .getExpressionLineage(rel, result)); + } + } + } + + @Test void testBudgetBoundaryAndMappingUnchanged() { + RexInputRef a = ref(0); + RexInputRef b = ref(1); + RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, b); + Map> mapping = mapping(a, b); + Map> original = new LinkedHashMap<>(mapping); + Set unlimited = expand(expression, mapping, -1); + assertNotNull(unlimited); + assertEquals(4, unlimited.size()); + assertEquals(unlimited, expand(expression, mapping, 4)); + assertNull(expand(expression, mapping, 3)); + assertEquals(original, mapping); + assertEquals(unlimited, expand(expression, mapping, 5)); + } + + @Test void testGeneratedProgramExceedsBudget() throws Exception { + RelNode rel = ExpressionLineageProgram.rel(16, 4, true); + Set lineage = rel.getCluster().getMetadataQuery().getExpressionLineage(rel, + RexInputRef.of(4, rel.getRowType().getFieldList())); + assertTrue(lineage == null, () -> "Expected unknown lineage, not " + + (lineage == null ? "null" : lineage.size()) + " candidates"); + } + + @Test void testBudgetCountsAttemptsBeforeDeduplication() { + RexInputRef a = ref(0); + RexInputRef b = ref(1); + RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, a, b); + Map> mapping = mapping(a, b); + // Commutative normalization merges 1 + 10 and 10 + 1, but constructing + // both still takes work. Budgeting the result Set size would miss this. + Set unlimited = expand(expression, mapping, -1); + assertNotNull(unlimited); + assertEquals(3, unlimited.size()); + assertEquals(unlimited, expand(expression, mapping, 4)); + assertNull(expand(expression, mapping, 3)); + } + + @Test void testRepeatedAndUnusedReferences() { + RexInputRef a = ref(0); + RexInputRef b = ref(1); + RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, a); + Map> mapping = mapping(a, b); + // Only a is referenced: repeated occurrences share a substitution, and + // unused b must not contribute a dimension to the product. + assertEquals(expand(expression, mapping, -1), expand(expression, mapping, 2)); + assertNull(expand(expression, mapping, 1)); + } + + @Test void testZeroBudgetAndConstants() { + Map> mapping = mapping(ref(0), ref(1)); + assertNull(expand(ref(0), mapping, 0)); + RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE); + assertEquals(ImmutableSet.of(one), expand(one, mapping, 0)); + assertThrows(IllegalArgumentException.class, () -> expand(one, mapping, -2)); + } + + @Test void testProductDoesNotOverflowOrEnumerate() { + RexInputRef a = ref(0); + RexInputRef b = ref(1); + RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, b); + Set many = new AbstractSet() { + @Override public int size() { + return 65_536; + } + + @Override public Iterator iterator() { + throw new AssertionError("Over-budget candidates must not be enumerated"); + } + }; + Map> mapping = new LinkedHashMap<>(); + mapping.put(a, many); + mapping.put(b, many); + // 65,536 squared wraps to zero in int arithmetic. + assertNull(expand(expression, mapping, Integer.MAX_VALUE)); + assertEquals(2, mapping.size()); + } + + @Test void testGeneratorRejectsInvalidDimensions() { + assertThrows(IllegalArgumentException.class, () -> ExpressionLineageProgram.sql(0, 4, true)); + assertThrows(IllegalArgumentException.class, () -> ExpressionLineageProgram.sql(2, 0, false)); + } + + @Test void testAllPredicatesUsesTheSameBudget() throws Exception { + for (int branches : new int[] {2, 16}) { + RelNode rel = ExpressionLineageProgram.rel(branches, 4, false); + RexBuilder builder = rel.getCluster().getRexBuilder(); + RexNode expression = RexInputRef.of(0, rel.getRowType().getFieldList()); + for (int i = 1; i < 4; i++) { + expression = builder.makeCall(SqlStdOperatorTable.MINUS, expression, + RexInputRef.of(i, rel.getRowType().getFieldList())); + } + RelNode filter = LogicalFilter.create(rel, builder.makeCall(SqlStdOperatorTable.GREATER_THAN, + expression, builder.makeExactLiteral(BigDecimal.ZERO))); + RelOptPredicateList predicates = filter.getCluster().getMetadataQuery() + .getAllPredicates(filter); + if (branches == 2) { + assertNotNull(predicates); + assertEquals(16, predicates.pulledUpPredicates.size()); + } else { + assertNull(predicates); + } + } + } + + @Test void testGeneratedMappingsRespectBudget() { + Random random = new Random(0); + for (int trial = 0; trial < 100; trial++) { + int columns = 1 + random.nextInt(4); + Map> mapping = new LinkedHashMap<>(); + RexNode expression = ref(0); + int product = 1; + for (int column = 0; column < columns; column++) { + int count = 1 + random.nextInt(4); + ImmutableSet.Builder values = ImmutableSet.builder(); + for (int value = 0; value < count; value++) { + values.add(rexBuilder.makeExactLiteral(BigDecimal.valueOf(value))); + } + mapping.put(ref(column), values.build()); + product *= count; + if (column > 0) { + expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, expression, ref(column)); + } + } + Map> original = new LinkedHashMap<>(mapping); + assertEquals(expand(expression, mapping, -1), expand(expression, mapping, product)); + assertNull(expand(expression, mapping, product - 1)); + assertEquals(original, mapping); + } + } + + private RexInputRef ref(int index) { + return rexBuilder.makeInputRef(rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER), + index); + } + + private Map> mapping(RexInputRef a, RexInputRef b) { + Map> mapping = new LinkedHashMap<>(); + Set replacements = ImmutableSet.of( + rexBuilder.makeExactLiteral(BigDecimal.ONE), rexBuilder.makeExactLiteral(BigDecimal.TEN)); + mapping.put(a, replacements); + mapping.put(b, replacements); + return mapping; + } + + private @Nullable Set expand(RexNode expression, Map> mapping, + int budget) { + return RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, expression, mapping, + budget); + } +} From f1a0f897c4875ab1bb4be47f510b78b6e1762eb7 Mon Sep 17 00:00:00 2001 From: ShengHuang Date: Sat, 7 Nov 2026 16:31:34 +0800 Subject: [PATCH 2/2] [CALCITE-7924] Simplify expression lineage budget tests --- .../rel/metadata/RelMdExpressionLineage.java | 6 +- .../metadata/ExpressionLineageProgram.java | 83 ------ .../metadata/RelMdExpressionLineageTest.java | 259 ++++++------------ 3 files changed, 85 insertions(+), 263 deletions(-) delete mode 100644 core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java index d3ade7696c93..7890582c049b 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -522,9 +522,7 @@ protected RelMdExpressionLineage() {} } if (maxExpansions >= 0) { - // Every referenced column adds a dimension to the Cartesian product. - // Count before copying RexNodes, even if equal candidates would later - // collapse in a Set. Division avoids overflow for very large products. + // Estimate the number of candidates before copying any RexNodes. int remaining = maxExpansions; for (Map.Entry> entry : mapping.entrySet()) { if (predFieldsUsed.get(entry.getKey().getIndex())) { @@ -533,6 +531,8 @@ protected RelMdExpressionLineage() {} return null; } if (count > 0) { + // Divide the remaining budget instead of multiplying the counts: + // the product can overflow even when each count fits in an int. remaining /= count; } } diff --git a/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java b/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java deleted file mode 100644 index 2aca64b2d5c3..000000000000 --- a/core/src/test/java/org/apache/calcite/rel/metadata/ExpressionLineageProgram.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.rel.metadata; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.tools.Frameworks; -import org.apache.calcite.tools.Planner; - -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.google.common.base.Preconditions.checkArgument; - -/** Generates storage-free SQL programs whose expression lineage has - * {@code branches ^ columns} candidate substitutions. Shared by regression - * tests and standalone measurements. */ -final class ExpressionLineageProgram { - private ExpressionLineageProgram() { - } - - /** Generates a SQL program with passthrough columns and a final computed - * column, either an arithmetic expression or a window expression. */ - static String sql(int branches, int columns, boolean window) { - checkArgument(branches >= 1, "branches must be positive"); - checkArgument(columns >= 1, "columns must be positive"); - String fields = IntStream.range(0, columns).mapToObj(i -> "c" + i) - .collect(Collectors.joining(", ")); - String expression; - if (window) { - String partition = columns == 1 ? "" : "partition by " - + IntStream.range(0, columns - 1).mapToObj(i -> "c" + i) - .collect(Collectors.joining(", ")) + " "; - expression = "row_number() over (" + partition + "order by c" + (columns - 1) + ")"; - } else { - expression = IntStream.range(0, columns).mapToObj(i -> "c" + i) - .collect(Collectors.joining(" + ")); - } - String inputs = IntStream.range(0, branches) - .mapToObj(i -> "select " + fields + " from events") - .collect(Collectors.joining(" union all ")); - return "select " + fields + ", " + expression + " as computed_value from (" + inputs + ") u"; - } - - /** Parses and validates the generated SQL using only Calcite. */ - static RelNode rel(int branches, int columns, boolean window) throws Exception { - String sql = sql(branches, columns, window); - SchemaPlus schema = Frameworks.createRootSchema(true); - schema.add("EVENTS", new AbstractTable() { - @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { - RelDataTypeFactory.Builder builder = typeFactory.builder(); - for (int i = 0; i < columns; i++) { - builder.add("C" + i, SqlTypeName.INTEGER); - } - return builder.build(); - } - }); - try (Planner planner = Frameworks.getPlanner(Frameworks.newConfigBuilder() - .defaultSchema(schema).build())) { - SqlNode validated = planner.validate(planner.parse(sql)); - return planner.rel(validated).project(); - } - } -} diff --git a/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java index 671f7fae8338..58f6b5baf4d1 100644 --- a/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java +++ b/core/src/test/java/org/apache/calcite/rel/metadata/RelMdExpressionLineageTest.java @@ -16,228 +16,133 @@ */ package org.apache.calcite.rel.metadata; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.plan.RelOptPredicateList; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Calc; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; -import com.google.common.collect.ImmutableSet; - -import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import java.math.BigDecimal; import java.util.AbstractSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.google.common.base.Preconditions.checkArgument; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; /** Tests for bounded expression-lineage expansion. */ class RelMdExpressionLineageTest { - private final RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl()); - @ParameterizedTest - @CsvSource({"1, 1", "2, 2", "2, 4", "4, 4", "8, 4", "16, 4", "90, 4", "2, 16"}) - void testGeneratedPrograms(int branches, int columns) throws Exception { - for (boolean window : new boolean[] {false, true}) { - for (boolean calc : new boolean[] {false, true}) { - RelNode rel = ExpressionLineageProgram.rel(branches, columns, window); - if (calc) { - HepPlanner planner = new HepPlanner(new HepProgramBuilder() - .addRuleInstance(CoreRules.PROJECT_TO_CALC).build()); - planner.setRoot(rel); - rel = planner.findBestExp(); - assertTrue(rel instanceof Calc, "Expected the Calc metadata path"); - } - RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); - RexNode result = RexInputRef.of(columns, rel.getRowType().getFieldList()); - Set lineage = mq.getExpressionLineage(rel, result); - if (Math.pow(branches, columns) > 10_000) { - // No truncated subset may escape as complete metadata. - assertNull(lineage); - } else { - assertNotNull(lineage); - assertEquals((int) Math.pow(branches, columns), lineage.size()); - } - // An expensive result must not poison other columns or later queries. - Set passthrough = mq.getExpressionLineage(rel, - RexInputRef.of(0, rel.getRowType().getFieldList())); - assertNotNull(passthrough); - assertEquals(branches, passthrough.size()); - assertEquals(lineage, mq.getExpressionLineage(rel, result)); - rel.getCluster().invalidateMetadataQuery(); - assertEquals(lineage, rel.getCluster().getMetadataQuery() - .getExpressionLineage(rel, result)); - } + @CsvSource({"2, 2, false", "10, 4, true", "11, 4, true", "11, 4, false"}) + void testUnionLineage(int branches, int columns, boolean window) throws Exception { + RelNode rel = ExpressionLineageProgram.rel(branches, columns, window); + RexNode computed = RexInputRef.of(columns, rel.getRowType().getFieldList()); + Set lineage = rel.getCluster().getMetadataQuery().getExpressionLineage(rel, computed); + int candidates = (int) Math.pow(branches, columns); + int budget = CalciteSystemProperty.EXPRESSION_LINEAGE_MAX_EXPANSIONS.value(); + if (budget >= 0 && candidates > budget) { + // Metadata was not computed. + assertNull(lineage); + } else { + assertNotNull(lineage); + assertEquals(candidates, lineage.size()); } } - @Test void testBudgetBoundaryAndMappingUnchanged() { - RexInputRef a = ref(0); - RexInputRef b = ref(1); - RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, b); - Map> mapping = mapping(a, b); - Map> original = new LinkedHashMap<>(mapping); - Set unlimited = expand(expression, mapping, -1); - assertNotNull(unlimited); - assertEquals(4, unlimited.size()); - assertEquals(unlimited, expand(expression, mapping, 4)); - assertNull(expand(expression, mapping, 3)); - assertEquals(original, mapping); - assertEquals(unlimited, expand(expression, mapping, 5)); - } - - @Test void testGeneratedProgramExceedsBudget() throws Exception { - RelNode rel = ExpressionLineageProgram.rel(16, 4, true); - Set lineage = rel.getCluster().getMetadataQuery().getExpressionLineage(rel, - RexInputRef.of(4, rel.getRowType().getFieldList())); - assertTrue(lineage == null, () -> "Expected unknown lineage, not " - + (lineage == null ? "null" : lineage.size()) + " candidates"); - } - - @Test void testBudgetCountsAttemptsBeforeDeduplication() { - RexInputRef a = ref(0); - RexInputRef b = ref(1); - RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, a, b); - Map> mapping = mapping(a, b); - // Commutative normalization merges 1 + 10 and 10 + 1, but constructing - // both still takes work. Budgeting the result Set size would miss this. - Set unlimited = expand(expression, mapping, -1); - assertNotNull(unlimited); - assertEquals(3, unlimited.size()); - assertEquals(unlimited, expand(expression, mapping, 4)); - assertNull(expand(expression, mapping, 3)); - } - - @Test void testRepeatedAndUnusedReferences() { - RexInputRef a = ref(0); - RexInputRef b = ref(1); - RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, a); - Map> mapping = mapping(a, b); - // Only a is referenced: repeated occurrences share a substitution, and - // unused b must not contribute a dimension to the product. - assertEquals(expand(expression, mapping, -1), expand(expression, mapping, 2)); - assertNull(expand(expression, mapping, 1)); - } - - @Test void testZeroBudgetAndConstants() { - Map> mapping = mapping(ref(0), ref(1)); - assertNull(expand(ref(0), mapping, 0)); - RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE); - assertEquals(ImmutableSet.of(one), expand(one, mapping, 0)); - assertThrows(IllegalArgumentException.class, () -> expand(one, mapping, -2)); - } - - @Test void testProductDoesNotOverflowOrEnumerate() { - RexInputRef a = ref(0); - RexInputRef b = ref(1); - RexNode expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, a, b); - Set many = new AbstractSet() { + @Test void testOverflowDoesNotStartExpansion() { + RexBuilder builder = new RexBuilder(new JavaTypeFactoryImpl()); + RelDataType type = builder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + RexInputRef a = builder.makeInputRef(type, 0); + RexInputRef b = builder.makeInputRef(type, 1); + Set origins = new AbstractSet() { @Override public int size() { return 65_536; } @Override public Iterator iterator() { - throw new AssertionError("Over-budget candidates must not be enumerated"); + throw new AssertionError("Over-budget origins must not be enumerated"); } }; Map> mapping = new LinkedHashMap<>(); - mapping.put(a, many); - mapping.put(b, many); - // 65,536 squared wraps to zero in int arithmetic. - assertNull(expand(expression, mapping, Integer.MAX_VALUE)); - assertEquals(2, mapping.size()); + mapping.put(a, origins); + mapping.put(b, origins); + // 65,536 squared wraps to zero as an int. + assertNull( + RelMdExpressionLineage.createAllPossibleExpressions(builder, + builder.makeCall(SqlStdOperatorTable.MINUS, a, b), mapping, Integer.MAX_VALUE)); } - @Test void testGeneratorRejectsInvalidDimensions() { - assertThrows(IllegalArgumentException.class, () -> ExpressionLineageProgram.sql(0, 4, true)); - assertThrows(IllegalArgumentException.class, () -> ExpressionLineageProgram.sql(2, 0, false)); - } + /** Generates UNION ALL plans for expression-lineage tests. */ + private static final class ExpressionLineageProgram { + private ExpressionLineageProgram() { + } - @Test void testAllPredicatesUsesTheSameBudget() throws Exception { - for (int branches : new int[] {2, 16}) { - RelNode rel = ExpressionLineageProgram.rel(branches, 4, false); - RexBuilder builder = rel.getCluster().getRexBuilder(); - RexNode expression = RexInputRef.of(0, rel.getRowType().getFieldList()); - for (int i = 1; i < 4; i++) { - expression = builder.makeCall(SqlStdOperatorTable.MINUS, expression, - RexInputRef.of(i, rel.getRowType().getFieldList())); - } - RelNode filter = LogicalFilter.create(rel, builder.makeCall(SqlStdOperatorTable.GREATER_THAN, - expression, builder.makeExactLiteral(BigDecimal.ZERO))); - RelOptPredicateList predicates = filter.getCluster().getMetadataQuery() - .getAllPredicates(filter); - if (branches == 2) { - assertNotNull(predicates); - assertEquals(16, predicates.pulledUpPredicates.size()); + /** Generates a query whose final column references every column of a UNION ALL. + * For example, {@code sql(2, 2, true)} generates: + *

{@code
+     * SELECT c0, c1, ROW_NUMBER() OVER (PARTITION BY c0 ORDER BY c1) AS computed_value
+     * FROM (SELECT c0, c1 FROM events UNION ALL SELECT c0, c1 FROM events) u
+     * }
*/ + static String sql(int branches, int columns, boolean window) { + checkArgument(branches >= 1, "branches must be positive"); + checkArgument(columns >= 1, "columns must be positive"); + String fields = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")); + String expression; + if (window) { + String partition = columns == 1 ? "" : "partition by " + + IntStream.range(0, columns - 1).mapToObj(i -> "c" + i) + .collect(Collectors.joining(", ")) + " "; + expression = "row_number() over (" + partition + "order by c" + (columns - 1) + ")"; } else { - assertNull(predicates); + expression = IntStream.range(0, columns).mapToObj(i -> "c" + i) + .collect(Collectors.joining(" + ")); } + String inputs = IntStream.range(0, branches) + .mapToObj(i -> "select " + fields + " from events") + .collect(Collectors.joining(" union all ")); + return "select " + fields + ", " + expression + " as computed_value from (" + inputs + ") u"; } - } - @Test void testGeneratedMappingsRespectBudget() { - Random random = new Random(0); - for (int trial = 0; trial < 100; trial++) { - int columns = 1 + random.nextInt(4); - Map> mapping = new LinkedHashMap<>(); - RexNode expression = ref(0); - int product = 1; - for (int column = 0; column < columns; column++) { - int count = 1 + random.nextInt(4); - ImmutableSet.Builder values = ImmutableSet.builder(); - for (int value = 0; value < count; value++) { - values.add(rexBuilder.makeExactLiteral(BigDecimal.valueOf(value))); - } - mapping.put(ref(column), values.build()); - product *= count; - if (column > 0) { - expression = rexBuilder.makeCall(SqlStdOperatorTable.MINUS, expression, ref(column)); + /** Returns the relational plan for {@link #sql(int, int, boolean)}. The lineage of + * its computed column has up to {@code branches ^ columns} candidate expressions. */ + static RelNode rel(int branches, int columns, boolean window) throws Exception { + String sql = sql(branches, columns, window); + SchemaPlus schema = Frameworks.createRootSchema(true); + schema.add("EVENTS", new AbstractTable() { + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (int i = 0; i < columns; i++) { + builder.add("C" + i, SqlTypeName.INTEGER); + } + return builder.build(); } + }); + try (Planner planner = Frameworks.getPlanner(Frameworks.newConfigBuilder() + .defaultSchema(schema).build())) { + SqlNode validated = planner.validate(planner.parse(sql)); + return planner.rel(validated).project(); } - Map> original = new LinkedHashMap<>(mapping); - assertEquals(expand(expression, mapping, -1), expand(expression, mapping, product)); - assertNull(expand(expression, mapping, product - 1)); - assertEquals(original, mapping); } } - - private RexInputRef ref(int index) { - return rexBuilder.makeInputRef(rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER), - index); - } - - private Map> mapping(RexInputRef a, RexInputRef b) { - Map> mapping = new LinkedHashMap<>(); - Set replacements = ImmutableSet.of( - rexBuilder.makeExactLiteral(BigDecimal.ONE), rexBuilder.makeExactLiteral(BigDecimal.TEN)); - mapping.put(a, replacements); - mapping.put(b, replacements); - return mapping; - } - - private @Nullable Set expand(RexNode expression, Map> mapping, - int budget) { - return RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, expression, mapping, - budget); - } }