From 924318b4db5e9b039dc999578f14ba97d37557d5 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Wed, 28 Oct 2026 14:07:25 +0800 Subject: [PATCH] [CALCITE-7895] Interpreter fails to execute plans containing EnumerableBindable inputs --- .../enumerable/EnumerableBindable.java | 17 +-- .../calcite/interpreter/Interpreter.java | 73 ++++++++++-- .../apache/calcite/test/InterpreterTest.java | 105 ++++++++++++++++++ 3 files changed, 173 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java index 666e8802d415..22e1ff4fbaf3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java @@ -21,9 +21,7 @@ import org.apache.calcite.interpreter.BindableRel; import org.apache.calcite.interpreter.Node; import org.apache.calcite.interpreter.Row; -import org.apache.calcite.interpreter.Sink; import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; @@ -39,8 +37,6 @@ import java.util.List; -import static java.util.Objects.requireNonNull; - /** * Relational expression that converts an enumerable input to interpretable * calling convention. @@ -73,16 +69,9 @@ protected EnumerableBindable(RelOptCluster cluster, RelNode input) { } @Override public Node implement(final InterpreterImplementor implementor) { - return () -> { - final Sink sink = - requireNonNull(implementor.relSinks.get(EnumerableBindable.this), - () -> "relSinks.get is null for " + EnumerableBindable.this).get(0); - final Enumerable<@Nullable Object[]> enumerable = bind(implementor.dataContext); - final Enumerator<@Nullable Object[]> enumerator = enumerable.enumerator(); - while (enumerator.moveNext()) { - sink.send(Row.asCopy(enumerator.current())); - } - }; + implementor.compiler.enumerable(this, + bind(implementor.dataContext).select(values -> Row.asCopy(values))); + return () -> { }; } /** diff --git a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java index 8bc82646a7ff..ff0957a89ad1 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java +++ b/core/src/main/java/org/apache/calcite/interpreter/Interpreter.java @@ -17,7 +17,8 @@ package org.apache.calcite.interpreter; import org.apache.calcite.DataContext; -import org.apache.calcite.DataContexts; +import org.apache.calcite.adapter.enumerable.EnumerableBindable; +import org.apache.calcite.adapter.enumerable.EnumerableInterpretable; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.linq4j.AbstractEnumerable; @@ -30,8 +31,11 @@ import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.AbstractRelNode; +import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -91,6 +95,17 @@ public Interpreter(DataContext dataContext, RelNode rootRel) { } private static RelNode optimize(RelNode rootRel) { + // The converters compile their inputs as Enumerable subtrees. Interpreter + // rules such as CALC_SPLIT can replace those inputs with logical nodes, + // which Enumerable code generation cannot handle. + rootRel = rootRel.accept(new RelHomogeneousShuttle() { + @Override public RelNode visit(RelNode other) { + if (isEnumerableBoundary(other)) { + return new EnumerableLeaf((InterpretableRel) other); + } + return super.visit(other); + } + }); final HepProgram hepProgram = new HepProgramBuilder() .addRuleInstance(CoreRules.CALC_SPLIT) .addRuleInstance(CoreRules.FILTER_SCAN) @@ -101,8 +116,41 @@ private static RelNode optimize(RelNode rootRel) { .build(); final HepPlanner planner = new HepPlanner(hepProgram); planner.setRoot(rootRel); - rootRel = planner.findBestExp(); - return rootRel; + return planner.findBestExp().accept(new RelHomogeneousShuttle() { + @Override public RelNode visit(RelNode other) { + if (other instanceof EnumerableLeaf) { + return ((EnumerableLeaf) other).rel; + } + return super.visit(other); + } + }); + } + + /** Converters that execute an Enumerable subtree without interpreter sources. + * Other InterpretableRel implementations may need their inputs compiled. */ + private static boolean isEnumerableBoundary(RelNode rel) { + return rel instanceof EnumerableBindable + || rel instanceof EnumerableInterpretable; + } + + /** Hides an Enumerable subtree during interpreter optimization. */ + private static class EnumerableLeaf extends AbstractRelNode { + private final InterpretableRel rel; + + EnumerableLeaf(InterpretableRel rel) { + super(rel.getCluster(), rel.getTraitSet()); + this.rel = rel; + } + + @Override protected RelDataType deriveRowType() { + return rel.getRowType(); + } + + @Override public RelWriter explainTerms(RelWriter pw) { + // Include the original identity so HepPlanner does not merge different + // subtrees hidden behind leaves with the same row type. + return super.explainTerms(pw).item("rel", rel.getId()); + } } @Override public Enumerator<@Nullable Object[]> enumerator() { @@ -324,6 +372,13 @@ Pair> visitRoot(RelNode p) { } @Override public void visit(RelNode p, int ordinal, @Nullable RelNode parent) { + if (isEnumerableBoundary(p)) { + // The converter executes its whole input subtree. Visiting its children + // here would also schedule them for interpreter execution. + final Node leafNode = implement((InterpretableRel) p); + requireNonNull(nodes.get(p)).node = leafNode; + return; + } for (;;) { rel = null; boolean found = dispatcher.invokeVisitor(this, p, REWRITE_METHOD_NAME); @@ -368,11 +423,7 @@ Pair> visitRoot(RelNode p) { boolean found = dispatcher.invokeVisitor(this, p, VISIT_METHOD_NAME); if (!found) { if (p instanceof InterpretableRel) { - InterpretableRel interpretableRel = (InterpretableRel) p; - node = - interpretableRel.implement( - new InterpretableRel.InterpreterImplementor(this, null, - DataContexts.EMPTY)); + node = implement((InterpretableRel) p); } else { // Probably need to add a visit(XxxRel) method to CoreCompiler. throw new AssertionError("interpreter: no implementation for " @@ -389,6 +440,12 @@ Pair> visitRoot(RelNode p) { } } + private Node implement(InterpretableRel rel) { + return rel.implement( + new InterpretableRel.InterpreterImplementor(this, null, + interpreter.dataContext)); + } + /** Fallback rewrite method. * *

Overriding methods (each with a different sub-class of {@link RelNode} diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java index ade1182ef171..b20f66c8bd8c 100644 --- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java +++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java @@ -18,19 +18,31 @@ import org.apache.calcite.DataContext; import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.EnumerableCalc; +import org.apache.calcite.adapter.enumerable.EnumerableInterpretable; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.enumerable.EnumerableValues; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.avatica.util.DateTimeUtils; +import org.apache.calcite.interpreter.Bindables; import org.apache.calcite.interpreter.Interpreter; import org.apache.calcite.linq4j.QueryProvider; +import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgramBuilder; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.schema.ScalarFunction; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.TableFunction; @@ -38,8 +50,10 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.schema.impl.TableFunctionImpl; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; @@ -215,6 +229,45 @@ private Sql sql(String sql) { return fixture().withSql(sql); } + private void checkEnumerableInputWithParameter(Function converter, + boolean project) { + final Function relFn = b -> { + final Values values = (Values) b.values(new String[] {"X"}, 1, 2, 3).build(); + final EnumerableValues input = + EnumerableValues.create(values.getCluster(), values.getRowType(), values.getTuples()); + final RexBuilder rexBuilder = b.getRexBuilder(); + final RelDataType type = b.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + final RexNode sequenceName = b.literal("interpreter_seq"); + final RexNode next = + rexBuilder.makeCall(type, SqlStdOperatorTable.NEXT_VALUE, ImmutableList.of(sequenceName)); + final RexProgramBuilder program = new RexProgramBuilder(input.getRowType(), rexBuilder); + program.addProject(rexBuilder.makeInputRef(input, 0), "X"); + program.addProject( + b.call(SqlStdOperatorTable.PLUS, next, + rexBuilder.makeDynamicParam(type, 0)), "Y"); + final EnumerableCalc calc = EnumerableCalc.create(input, program.getProgram()); + final RelNode converted = converter.apply(calc); + return project ? b.push(converted).project(b.field(1)).build() : converted; + }; + final RelNode rel = fixture().withRelBuilder(relFn); + final DataContext dataContext = new MyDataContext(rootSchema(), rel) { + @Override public @Nullable Object get(String name) { + return name.equals("?0") ? 7L : super.get(name); + } + }; + SqlFunctions.resetThreadSequences(); + try { + if (project) { + assertInterpret(rel, dataContext, true, "[8]", "[9]", "[10]"); + } else { + assertInterpret(rel, dataContext, true, "[1, 8]", "[2, 9]", "[3, 10]"); + } + assertThat(SqlFunctions.sequenceCurrentValue("interpreter_seq"), is(3L)); + } finally { + SqlFunctions.resetThreadSequences(); + } + } + private void reset() { rootSchema = Frameworks.createRootSchema(true); } @@ -815,4 +868,56 @@ private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) { fixture().withRel(relFn) .returnsRows("[]", "[]"); } + + /** Tests an Enumerable input with a dynamic parameter and a sequence + * ([CALCITE-7895]). */ + @Test void testInterpretEnumerableInputWithParameter() { + checkEnumerableInputWithParameter( + input -> requireNonNull(EnumerableRules.TO_BINDABLE.convert(input)), true); + } + + /** Tests an Enumerable subtree as the interpreter root + * ([CALCITE-7895]). */ + @Test void testInterpretEnumerableInterpretableRootWithParameter() { + checkEnumerableInputWithParameter(input -> + new EnumerableInterpretable(input.getCluster(), input) { }, false); + } + + /** Tests UNION ALL with different Enumerable inputs of the same row type + * ([CALCITE-7895]). */ + @Test void testInterpretUnionAllWithEnumerableInputs() { + final Function relFn = b -> { + for (int value : new int[] {1, 10}) { + final Values values = + (Values) b.values(new String[] {"X"}, value, value * 2).build(); + final EnumerableValues input = + EnumerableValues.create(values.getCluster(), values.getRowType(), values.getTuples()); + b.push(requireNonNull(EnumerableRules.TO_BINDABLE.convert(input))); + } + return b.union(true).build(); + }; + fixture().withRel(relFn) + .returnsRowsUnordered("[1]", "[2]", "[10]", "[20]"); + } + + /** Test case for + * [CALCITE-7895] + * Interpreter fails to execute plans containing EnumerableBindable inputs. + * + *

Checks that the plan includes an Enumerable-to-Bindable conversion. */ + @Test void testInterpretEnumerableSql() { + CalciteAssert.hr() + .query("select \"empid\" + 1 as x from \"hr\".\"emps\" order by x") + .withProperty(Hook.ENABLE_BINDABLE, true) + .withHook(Hook.PLANNER, (RelOptPlanner planner) -> { + planner.removeRule(Bindables.FROM_NONE_RULE); + planner.removeRule(Bindables.BINDABLE_PROJECT_RULE); + planner.removeRule(EnumerableRules.ENUMERABLE_SORT_RULE); + planner.removeRule(EnumerableRules.TO_INTERPRETER); + }) + .explainContains("BindableSort(sort0=[$0], dir0=[ASC])\n" + + " EnumerableBindable\n" + + " EnumerableCalc(") + .returns("X=101\nX=111\nX=151\nX=201\n"); + } }