Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@
import org.apache.calcite.interpreter.BindableRel;
import org.apache.calcite.interpreter.Node;
import org.apache.calcite.interpreter.Row;
import org.apache.calcite.interpreter.Sink;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.plan.ConventionTraitDef;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
Expand All @@ -39,8 +37,6 @@

import java.util.List;

import static java.util.Objects.requireNonNull;

/**
* Relational expression that converts an enumerable input to interpretable
* calling convention.
Expand Down Expand Up @@ -73,16 +69,9 @@
}

@Override public Node implement(final InterpreterImplementor implementor) {
return () -> {
final Sink sink =
requireNonNull(implementor.relSinks.get(EnumerableBindable.this),
() -> "relSinks.get is null for " + EnumerableBindable.this).get(0);
final Enumerable<@Nullable Object[]> enumerable = bind(implementor.dataContext);
final Enumerator<@Nullable Object[]> enumerator = enumerable.enumerator();
while (enumerator.moveNext()) {
sink.send(Row.asCopy(enumerator.current()));
}
};
implementor.compiler.enumerable(this,
bind(implementor.dataContext).select(values -> Row.asCopy(values)));

Check warning on line 73 in core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace this lambda with method reference 'Row::asCopy'.

See more on https://sonarcloud.io/project/issues?id=apache_calcite&issues=AaGB6ZfzLe9LSCHBU5DT&open=AaGB6ZfzLe9LSCHBU5DT&pullRequest=5367
return () -> { };
}

/**
Expand Down
73 changes: 65 additions & 8 deletions core/src/main/java/org/apache/calcite/interpreter/Interpreter.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
package org.apache.calcite.interpreter;

import org.apache.calcite.DataContext;
import org.apache.calcite.DataContexts;
import org.apache.calcite.adapter.enumerable.EnumerableBindable;
import org.apache.calcite.adapter.enumerable.EnumerableInterpretable;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.config.CalciteSystemProperty;
import org.apache.calcite.linq4j.AbstractEnumerable;
Expand All @@ -30,8 +31,11 @@
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelHomogeneousShuttle;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
Expand Down Expand Up @@ -91,6 +95,17 @@ public Interpreter(DataContext dataContext, RelNode rootRel) {
}

private static RelNode optimize(RelNode rootRel) {
// The converters compile their inputs as Enumerable subtrees. Interpreter
// rules such as CALC_SPLIT can replace those inputs with logical nodes,
// which Enumerable code generation cannot handle.
rootRel = rootRel.accept(new RelHomogeneousShuttle() {
@Override public RelNode visit(RelNode other) {
if (isEnumerableBoundary(other)) {
return new EnumerableLeaf((InterpretableRel) other);
}
return super.visit(other);
}
});
final HepProgram hepProgram = new HepProgramBuilder()
.addRuleInstance(CoreRules.CALC_SPLIT)
.addRuleInstance(CoreRules.FILTER_SCAN)
Expand All @@ -101,8 +116,41 @@ private static RelNode optimize(RelNode rootRel) {
.build();
final HepPlanner planner = new HepPlanner(hepProgram);
planner.setRoot(rootRel);
rootRel = planner.findBestExp();
return rootRel;
return planner.findBestExp().accept(new RelHomogeneousShuttle() {
@Override public RelNode visit(RelNode other) {
if (other instanceof EnumerableLeaf) {
return ((EnumerableLeaf) other).rel;
}
return super.visit(other);
}
});
}

/** Converters that execute an Enumerable subtree without interpreter sources.
* Other InterpretableRel implementations may need their inputs compiled. */
private static boolean isEnumerableBoundary(RelNode rel) {
return rel instanceof EnumerableBindable
|| rel instanceof EnumerableInterpretable;
}

/** Hides an Enumerable subtree during interpreter optimization. */
private static class EnumerableLeaf extends AbstractRelNode {
private final InterpretableRel rel;

EnumerableLeaf(InterpretableRel rel) {
super(rel.getCluster(), rel.getTraitSet());
this.rel = rel;
}

@Override protected RelDataType deriveRowType() {
return rel.getRowType();
}

@Override public RelWriter explainTerms(RelWriter pw) {
// Include the original identity so HepPlanner does not merge different
// subtrees hidden behind leaves with the same row type.
return super.explainTerms(pw).item("rel", rel.getId());
}
}

@Override public Enumerator<@Nullable Object[]> enumerator() {
Expand Down Expand Up @@ -324,6 +372,13 @@ Pair<RelNode, Map<RelNode, NodeInfo>> visitRoot(RelNode p) {
}

@Override public void visit(RelNode p, int ordinal, @Nullable RelNode parent) {
if (isEnumerableBoundary(p)) {
// The converter executes its whole input subtree. Visiting its children
// here would also schedule them for interpreter execution.
final Node leafNode = implement((InterpretableRel) p);
requireNonNull(nodes.get(p)).node = leafNode;
return;
}
for (;;) {
rel = null;
boolean found = dispatcher.invokeVisitor(this, p, REWRITE_METHOD_NAME);
Expand Down Expand Up @@ -368,11 +423,7 @@ Pair<RelNode, Map<RelNode, NodeInfo>> visitRoot(RelNode p) {
boolean found = dispatcher.invokeVisitor(this, p, VISIT_METHOD_NAME);
if (!found) {
if (p instanceof InterpretableRel) {
InterpretableRel interpretableRel = (InterpretableRel) p;
node =
interpretableRel.implement(
new InterpretableRel.InterpreterImplementor(this, null,
DataContexts.EMPTY));
node = implement((InterpretableRel) p);
} else {
// Probably need to add a visit(XxxRel) method to CoreCompiler.
throw new AssertionError("interpreter: no implementation for "
Expand All @@ -389,6 +440,12 @@ Pair<RelNode, Map<RelNode, NodeInfo>> visitRoot(RelNode p) {
}
}

private Node implement(InterpretableRel rel) {
return rel.implement(
new InterpretableRel.InterpreterImplementor(this, null,
interpreter.dataContext));
}

/** Fallback rewrite method.
*
* <p>Overriding methods (each with a different sub-class of {@link RelNode}
Expand Down
105 changes: 105 additions & 0 deletions core/src/test/java/org/apache/calcite/test/InterpreterTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,42 @@

import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.enumerable.EnumUtils;
import org.apache.calcite.adapter.enumerable.EnumerableCalc;
import org.apache.calcite.adapter.enumerable.EnumerableInterpretable;
import org.apache.calcite.adapter.enumerable.EnumerableRules;
import org.apache.calcite.adapter.enumerable.EnumerableValues;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.avatica.util.DateTimeUtils;
import org.apache.calcite.interpreter.Bindables;
import org.apache.calcite.interpreter.Interpreter;
import org.apache.calcite.linq4j.QueryProvider;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.core.Values;
import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexProgramBuilder;
import org.apache.calcite.runtime.Hook;
import org.apache.calcite.runtime.SqlFunctions;
import org.apache.calcite.schema.ScalarFunction;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.schema.TableFunction;
import org.apache.calcite.schema.impl.AbstractSchema;
import org.apache.calcite.schema.impl.ScalarFunctionImpl;
import org.apache.calcite.schema.impl.TableFunctionImpl;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
Expand Down Expand Up @@ -215,6 +229,45 @@ private Sql sql(String sql) {
return fixture().withSql(sql);
}

private void checkEnumerableInputWithParameter(Function<RelNode, RelNode> converter,
boolean project) {
final Function<RelBuilder, RelNode> relFn = b -> {
final Values values = (Values) b.values(new String[] {"X"}, 1, 2, 3).build();
final EnumerableValues input =
EnumerableValues.create(values.getCluster(), values.getRowType(), values.getTuples());
final RexBuilder rexBuilder = b.getRexBuilder();
final RelDataType type = b.getTypeFactory().createSqlType(SqlTypeName.BIGINT);
final RexNode sequenceName = b.literal("interpreter_seq");
final RexNode next =
rexBuilder.makeCall(type, SqlStdOperatorTable.NEXT_VALUE, ImmutableList.of(sequenceName));
final RexProgramBuilder program = new RexProgramBuilder(input.getRowType(), rexBuilder);
program.addProject(rexBuilder.makeInputRef(input, 0), "X");
program.addProject(
b.call(SqlStdOperatorTable.PLUS, next,
rexBuilder.makeDynamicParam(type, 0)), "Y");
final EnumerableCalc calc = EnumerableCalc.create(input, program.getProgram());
final RelNode converted = converter.apply(calc);
return project ? b.push(converted).project(b.field(1)).build() : converted;
};
final RelNode rel = fixture().withRelBuilder(relFn);
final DataContext dataContext = new MyDataContext(rootSchema(), rel) {
@Override public @Nullable Object get(String name) {
return name.equals("?0") ? 7L : super.get(name);
}
};
SqlFunctions.resetThreadSequences();
try {
if (project) {
assertInterpret(rel, dataContext, true, "[8]", "[9]", "[10]");
} else {
assertInterpret(rel, dataContext, true, "[1, 8]", "[2, 9]", "[3, 10]");
}
assertThat(SqlFunctions.sequenceCurrentValue("interpreter_seq"), is(3L));
} finally {
SqlFunctions.resetThreadSequences();
}
}

private void reset() {
rootSchema = Frameworks.createRootSchema(true);
}
Expand Down Expand Up @@ -815,4 +868,56 @@ private static RelNode applyAggregateRemoveLiteralAggRule(RelNode rel) {
fixture().withRel(relFn)
.returnsRows("[]", "[]");
}

/** Tests an Enumerable input with a dynamic parameter and a sequence
* (<a href="https://issues.apache.org/jira/browse/CALCITE-7895">[CALCITE-7895]</a>). */
@Test void testInterpretEnumerableInputWithParameter() {
checkEnumerableInputWithParameter(
input -> requireNonNull(EnumerableRules.TO_BINDABLE.convert(input)), true);
}

/** Tests an Enumerable subtree as the interpreter root
* (<a href="https://issues.apache.org/jira/browse/CALCITE-7895">[CALCITE-7895]</a>). */
@Test void testInterpretEnumerableInterpretableRootWithParameter() {
checkEnumerableInputWithParameter(input ->
new EnumerableInterpretable(input.getCluster(), input) { }, false);
}

/** Tests UNION ALL with different Enumerable inputs of the same row type
* (<a href="https://issues.apache.org/jira/browse/CALCITE-7895">[CALCITE-7895]</a>). */
@Test void testInterpretUnionAllWithEnumerableInputs() {
final Function<RelBuilder, RelNode> relFn = b -> {
for (int value : new int[] {1, 10}) {
final Values values =
(Values) b.values(new String[] {"X"}, value, value * 2).build();
final EnumerableValues input =
EnumerableValues.create(values.getCluster(), values.getRowType(), values.getTuples());
b.push(requireNonNull(EnumerableRules.TO_BINDABLE.convert(input)));
}
return b.union(true).build();
};
fixture().withRel(relFn)
.returnsRowsUnordered("[1]", "[2]", "[10]", "[20]");
}

/** Test case for
* <a href="https://issues.apache.org/jira/browse/CALCITE-7895">[CALCITE-7895]
* Interpreter fails to execute plans containing EnumerableBindable inputs</a>.
*
* <p>Checks that the plan includes an Enumerable-to-Bindable conversion. */
@Test void testInterpretEnumerableSql() {
CalciteAssert.hr()
.query("select \"empid\" + 1 as x from \"hr\".\"emps\" order by x")
.withProperty(Hook.ENABLE_BINDABLE, true)
.withHook(Hook.PLANNER, (RelOptPlanner planner) -> {
planner.removeRule(Bindables.FROM_NONE_RULE);
planner.removeRule(Bindables.BINDABLE_PROJECT_RULE);
planner.removeRule(EnumerableRules.ENUMERABLE_SORT_RULE);
planner.removeRule(EnumerableRules.TO_INTERPRETER);
})
.explainContains("BindableSort(sort0=[$0], dir0=[ASC])\n"
+ " EnumerableBindable\n"
+ " EnumerableCalc(")
.returns("X=101\nX=111\nX=151\nX=201\n");
}
}
Loading