diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsQueryDslBuilder.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsQueryDslBuilder.java index 2b923d2e427d0a..c82bca90d16d4c 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsQueryDslBuilder.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsQueryDslBuilder.java @@ -177,8 +177,7 @@ public static EsQueryDslResult buildQueryDslWithResult(ConnectorExpression filte } private static boolean referencesUnsafeSourceField(ConnectorExpression expression, Set unsafeFields) { - if (expression instanceof ConnectorFunctionCall - && "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName())) { + if (isEsquery(expression)) { // esquery explicitly requests ES index semantics rather than SQL source-value semantics. return false; } @@ -194,8 +193,7 @@ private static boolean referencesUnsafeSourceField(ConnectorExpression expressio } private static void requireLocalEvaluation(ConnectorExpression expression) { - if (expression instanceof ConnectorFunctionCall - && "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName())) { + if (isEsquery(expression)) { // esquery has no BE implementation. Never leave it inside a residual OR/NOT that // cannot be translated exactly; a remote match_all cannot make it locally evaluable. throw new DorisConnectorException("Cannot evaluate esquery locally: its containing Boolean expression " @@ -664,10 +662,16 @@ private static String convertLikeToWildcard(String pattern) { // Function call (esquery) // ----------------------------------------------------------------------- + /** The builtin esquery(field, dsl), whose DSL Elasticsearch runs; a UDF of that name is not it. */ + private static boolean isEsquery(ConnectorExpression expression) { + return expression instanceof ConnectorFunctionCall + && ((ConnectorFunctionCall) expression).isBuiltin() + && "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName()); + } + private static String functionCallToDsl(ConnectorFunctionCall funcCall, List notPushDownList) { - if ("esquery".equals(funcCall.getFunctionName()) - && funcCall.getArguments().size() == 2) { + if (isEsquery(funcCall) && funcCall.getArguments().size() == 2) { ConnectorExpression secondArg = funcCall.getArguments().get(1); if (secondArg instanceof ConnectorLiteral) { Object val = ((ConnectorLiteral) secondArg).getValue(); diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsQueryDslBuilderTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsQueryDslBuilderTest.java index 4a5798677b20a9..d8135555521773 100644 --- a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsQueryDslBuilderTest.java +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsQueryDslBuilderTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.es; import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.pushdown.ConnectorAnd; import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; import org.apache.doris.connector.spi.pushdown.ConnectorComparison; @@ -33,6 +34,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; public class EsQueryDslBuilderTest { @@ -59,6 +61,27 @@ void partialAndRetainsOnlyItsUnsupportedConjunct() { Assertions.assertEquals("{\"term\":{\"name\":\"one\"}}", result.getQueryDsl()); } + @Test + void udfNamedEsqueryIsEvaluatedByDoris() { + List args = Arrays.asList(new ConnectorColumnRef("name", ConnectorType.of("VARCHAR")), + ConnectorLiteral.ofString("{\"term\":{\"name\":\"one\"}}")); + ConnectorExpression esquery = new ConnectorFunctionCall("esquery", ConnectorType.of("BOOLEAN"), args); + ConnectorExpression udf = new ConnectorFunctionCall("esquery", ConnectorType.of("BOOLEAN"), args, false); + EsQueryDslResult pushed = buildWithTracking(esquery); + Assertions.assertEquals("{\"term\":{\"name\":\"one\"}}", pushed.getQueryDsl()); + Assertions.assertTrue(pushed.getNotPushedIndices().isEmpty()); + EsQueryDslResult local = buildWithTracking(udf); + Assertions.assertEquals(EsQueryDslBuilder.matchAllQuery(), local.getQueryDsl()); + Assertions.assertEquals(Collections.singletonList(0), local.getNotPushedIndices()); + // In an OR that cannot be pushed, the builtin has no BE implementation to fall back to; the UDF has. + ConnectorExpression unpushable = new ConnectorFunctionCall("unsupported_predicate", ConnectorType.of("BOOLEAN"), + Collections.singletonList(new ConnectorColumnRef("name", ConnectorType.of("VARCHAR")))); + Assertions.assertThrows(DorisConnectorException.class, + () -> buildWithTracking(new ConnectorOr(Arrays.asList(esquery, unpushable)))); + Assertions.assertEquals(Collections.singletonList(0), + buildWithTracking(new ConnectorOr(Arrays.asList(udf, unpushable))).getNotPushedIndices()); + } + @Test void orWithPartiallyPushableAndRemainsLocal() { EsQueryDslResult result = buildWithTracking(new ConnectorOr( diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcQueryBuilder.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcQueryBuilder.java index 59490ca593260b..cbc2828041a949 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcQueryBuilder.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcQueryBuilder.java @@ -764,6 +764,11 @@ private String betweenToSql(ConnectorBetween between, Map colMap private static final Set UNARY_OPERATORS = Collections.singleton("~"); private String functionCallToSql(ConnectorFunctionCall func, Map colMapping) { + // A UDF runs only in Doris: the remote database has no such function, or one of the same name that + // computes something else. This also keeps a pre-rendered fragment that calls one local. + if (!func.isBuiltin()) { + return null; + } String funcName = func.getFunctionName(); // 0. Handle arithmetic operators rendered as infix: (arg1 + arg2) diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcQueryBuilderTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcQueryBuilderTest.java index 0df3d0807ef1c6..2c27b29c59f022 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcQueryBuilderTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcQueryBuilderTest.java @@ -199,6 +199,29 @@ void testPreRenderedOperandIsGrouped() { ConnectorComparison.Operator.EQ, fragment, ConnectorLiteral.ofBoolean(false))), -1)); } + @Test + void testUserDefinedFunctionsRemainLocalWithoutLimit() { + JdbcFunctionPushdownConfig config = JdbcFunctionPushdownConfig.create(JdbcDbType.MYSQL, null, true); + JdbcQueryBuilder builder = new JdbcQueryBuilder(JdbcDbType.MYSQL, config, Collections.emptyMap()); + List args = Collections.singletonList(new ConnectorColumnRef("id", INT_TYPE)); + // A UDF named like MySQL's abs, and a pre-rendered fragment that calls a UDF. + for (ConnectorExpression udf : Arrays.asList(new ConnectorFunctionCall("abs", INT_TYPE, args, false), + new ConnectorFunctionCall("CASE WHEN my_udf(`id`) THEN 1 END", INT_TYPE, + Collections.emptyList(), false))) { + ConnectorExpression predicate = new ConnectorComparison( + ConnectorComparison.Operator.EQ, udf, ConnectorLiteral.ofInt(1)); + Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl`", + builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of(predicate), 1)); + Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl` WHERE (`id` = 1)", + builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of( + new ConnectorAnd(Arrays.asList(predicate, simpleComparison("id", 1)))), 1)); + } + Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl` WHERE ((abs(`id`)) = 1) LIMIT 1", + builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of(new ConnectorComparison( + ConnectorComparison.Operator.EQ, new ConnectorFunctionCall("abs", INT_TYPE, args), + ConnectorLiteral.ofInt(1))), 1)); + } + @Test void testEmptyInRemainsLocalWithoutLimit() { ConnectorColumnRef column = new ConnectorColumnRef("id", INT_TYPE); diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCall.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCall.java index 6c8d14a5565dd9..b261c891fd1538 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCall.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCall.java @@ -26,7 +26,7 @@ /** * A function call expression (catch-all for unsupported function types). * Connector plugins can inspect the function name and arguments to decide - * whether to push down or reject. + * whether to push down or reject; only a call that {@link #isBuiltin() is a builtin} may be matched by name. */ public final class ConnectorFunctionCall implements ConnectorExpression { @@ -35,21 +35,41 @@ public final class ConnectorFunctionCall implements ConnectorExpression { private final String functionName; private final ConnectorType returnType; private final List arguments; + private final boolean builtin; + /** A call of a Doris builtin function or operator. */ public ConnectorFunctionCall(String functionName, ConnectorType returnType, List arguments) { + this(functionName, returnType, arguments, true); + } + + public ConnectorFunctionCall(String functionName, + ConnectorType returnType, + List arguments, + boolean builtin) { this.functionName = Objects.requireNonNull(functionName, "functionName"); this.returnType = Objects.requireNonNull(returnType, "returnType"); this.arguments = arguments == null ? Collections.emptyList() : Collections.unmodifiableList(arguments); + this.builtin = builtin; } public String getFunctionName() { return functionName; } + /** + * Whether this calls a Doris builtin function or operator. A user-defined function (a Java or Python UDF) + * does not: its name says nothing about what it computes, so a connector must never translate it by name. + * For the fallback carrier (Rule 6 of the {@code pushdown} package), false when the expression it renders + * calls a user-defined function. + */ + public boolean isBuiltin() { + return builtin; + } + public ConnectorType getReturnType() { return returnType; } @@ -79,11 +99,12 @@ public boolean equals(Object o) { ConnectorFunctionCall that = (ConnectorFunctionCall) o; return functionName.equals(that.functionName) && returnType.equals(that.returnType) - && arguments.equals(that.arguments); + && arguments.equals(that.arguments) + && builtin == that.builtin; } @Override public int hashCode() { - return Objects.hash(functionName, returnType, arguments); + return Objects.hash(functionName, returnType, arguments, builtin); } } diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/package-info.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/package-info.java index 372e09d0b85e72..62ee088c117b31 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/package-info.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/pushdown/package-info.java @@ -152,14 +152,22 @@ * original expression as the residual, so their predicates are also evaluated on BE. That is correct but not * free — and it is the reason a slightly-too-wide pushdown is survivable today.

* - *

Rule 6 — {@link ConnectorFunctionCall} is also the fallback carrier. Do not match it by name

- * - *

{@code ExprToConnectorExpressionConverter.fallback} turns any expression it does not model into a - * {@link ConnectorFunctionCall} whose name is the rendered Doris SQL text and whose argument list is - * empty. It is not a function call at all. Matching function names blindly will make a connector - * translate {@code my_udf(a, b) = 1} as a call to a function literally named {@code "my_udf(a, b) = 1"}. - * jdbc handles this deliberately: an argument-less call whose name is not a plain identifier is emitted as a - * pre-rendered SQL fragment ({@code JdbcQueryBuilder}).

+ *

Rule 6 — {@link ConnectorFunctionCall} is also the fallback carrier and the user-defined function. + * Match it by name only when it is a builtin

+ * + *

{@code ExprToConnectorExpressionConverter.fallback} turns any expression it does not model, such as a + * {@code CASE} expression, into a {@link ConnectorFunctionCall} whose name is the rendered Doris SQL text + * and whose argument list is empty. It is not a function call at all. Matching function names blindly + * will make a connector translate {@code CASE WHEN a > 0 THEN b END} as a call to a function literally named + * by that text. jdbc handles this deliberately: an argument-less call whose name is not a plain identifier is + * emitted as a pre-rendered SQL fragment ({@code JdbcQueryBuilder}).

+ * + *

A user-defined function (a Java or Python UDF) arrives with its own name and real arguments, exactly like + * a builtin, but with {@link ConnectorFunctionCall#isBuiltin()} false + * ({@code ExprToConnectorExpressionConverter.convertFunctionCall}). Its name says nothing about what it + * computes: a UDF may be named {@code starts_with} or {@code esquery}, and no remote system runs it. Never + * translate such a call. A fallback whose rendered expression calls one is not a builtin either, so a + * connector that emits fallbacks as SQL must check it too.

* *

Two related shapes to expect:

*
    @@ -168,8 +176,8 @@ * {@code ConnectorFunctionCall} named {@code like} with three arguments. A connector that only handles * {@link ConnectorLike} therefore never sees a custom escape character, and may assume the fixed * backslash escape documented on {@link ConnectorLike}. - *
  • Arithmetic and genuine scalar functions arrive as {@link ConnectorFunctionCall} with real arguments; - * those are safe to match by name.
  • + *
  • Arithmetic and builtin scalar functions arrive as {@link ConnectorFunctionCall} with real arguments and + * {@link ConnectorFunctionCall#isBuiltin()} true; those are safe to match by name.
  • *
* *

Anything you cannot interpret confidently falls under Rule 1: drop the conjunct.

diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index 73b1fd28d14c1d..b3ed765abd567e 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -88,8 +88,8 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException Assertions.assertNotNull(in, "missing connector plugin API version resource"); version.load(in); } - // Major 29 adds typed table-absence errors after the upstream API 28 changes. - Assertions.assertEquals("30.0", version.getProperty("api.version")); + // Major 31 adds ConnectorFunctionCall#isBuiltin, which tells a UDF from a builtin of the same name. + Assertions.assertEquals("31.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCallTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCallTest.java new file mode 100644 index 00000000000000..7a8198ac766d02 --- /dev/null +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/pushdown/ConnectorFunctionCallTest.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.spi.pushdown; + +import org.apache.doris.connector.spi.ConnectorType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +public class ConnectorFunctionCallTest { + private static final ConnectorType BOOLEAN = ConnectorType.of("BOOLEAN"); + private static final List ARGS = Collections.singletonList(ConnectorLiteral.ofString("a")); + + @Test + public void callIsBuiltinUnlessCreatedAsUdf() { + Assertions.assertTrue(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS).isBuiltin()); + Assertions.assertTrue(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, true).isBuiltin()); + Assertions.assertFalse(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, false).isBuiltin()); + } + + @Test + public void udfDiffersFromTheBuiltinOfTheSameName() { + ConnectorFunctionCall builtin = new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS); + ConnectorFunctionCall alsoBuiltin = new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, true); + Assertions.assertEquals(builtin, alsoBuiltin); + Assertions.assertEquals(builtin.hashCode(), alsoBuiltin.hashCode()); + Assertions.assertNotEquals(builtin, new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, false)); + } +} diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index f6ff0f9aeda315..0ab1906e4f4a42 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 30.0 + 31.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java index 36cd21e3194084..8eccb3b9935c2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java @@ -39,6 +39,7 @@ import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Function; import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.StructField; @@ -114,10 +115,10 @@ public static ConnectorExpression convert(Expr expr) { String fnName = fce.getFnName().getFunction(); // Nereids translates Like/Regexp to FunctionCallExpr via visitScalarFunction, // so we need to detect them here and convert to ConnectorLike. - if ("like".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) { + if (isBuiltin(fce) && "like".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) { return new ConnectorLike(ConnectorLike.Operator.LIKE, convert(fce.getChild(0)), convert(fce.getChild(1))); - } else if ("regexp".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) { + } else if (isBuiltin(fce) && "regexp".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) { return new ConnectorLike(ConnectorLike.Operator.REGEXP, convert(fce.getChild(0)), convert(fce.getChild(1))); } @@ -336,7 +337,12 @@ private static ConnectorExpression convertFunctionCall(FunctionCallExpr fce) { List args = fce.getChildren().stream() .map(ExprToConnectorExpressionConverter::convert) .collect(Collectors.toList()); - return new ConnectorFunctionCall(fnName, retType, args); + // A UDF keeps only its bare name here, which may be a builtin's; the flag is what tells them apart. + return new ConnectorFunctionCall(fnName, retType, args, isBuiltin(fce)); + } + + private static boolean isBuiltin(FunctionCallExpr fce) { + return fce.getFn().getBinaryType() == Function.BinaryType.BUILTIN; } private static ConnectorExpression fallback(Expr expr) { @@ -351,7 +357,9 @@ private static ConnectorExpression fallback(Expr expr) { } catch (Exception e) { sql = expr.debugString(); } - return new ConnectorFunctionCall(sql, retType, Collections.emptyList()); + boolean callsUdf = expr.contains( + (Expr node) -> node instanceof FunctionCallExpr && !isBuiltin((FunctionCallExpr) node)); + return new ConnectorFunctionCall(sql, retType, Collections.emptyList(), !callsUdf); } private static ConnectorType scalarTypeToConnectorType(ScalarType st) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ExprToConnectorFunctionCallTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ExprToConnectorFunctionCallTest.java new file mode 100644 index 00000000000000..a334c86d7dd63e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ExprToConnectorFunctionCallTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.connector.converter; + +import org.apache.doris.analysis.CaseExpr; +import org.apache.doris.analysis.CaseWhenClause; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionName; +import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.Type; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.pushdown.ConnectorFunctionCall; +import org.apache.doris.connector.spi.pushdown.ConnectorLike; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.stream.Collectors; + +/** + * Connectors match a call by its name, and a UDF may carry any name, a builtin's included; the converter has to + * say which calls are builtins. + */ +class ExprToConnectorFunctionCallTest { + + private static Expr varchar(String value) { + return new VarcharLiteral(value).toLegacyLiteral(); + } + + private static Expr integer(int value) { + return new IntegerLiteral(value).toLegacyLiteral(); + } + + /** A call as Nereids translates it: the catalog function, a UDF's under its database, and its arguments. */ + private static FunctionCallExpr call(String name, Function.BinaryType binaryType, Expr... args) { + FunctionName functionName = binaryType == Function.BinaryType.BUILTIN + ? new FunctionName(name) : new FunctionName("udf_db", name); + ScalarFunction function = new ScalarFunction(functionName, + Arrays.stream(args).map(Expr::getType).collect(Collectors.toList()), Type.BOOLEAN, false, + binaryType, true, true); + return new FunctionCallExpr(function, new FunctionParams(false, Arrays.asList(args)), true); + } + + @Test + void udfsAreFlaggedWhereTheBuiltinOfTheSameNameIsNot() { + ConnectorFunctionCall builtin = (ConnectorFunctionCall) ExprToConnectorExpressionConverter.convert( + call("starts_with", Function.BinaryType.BUILTIN, varchar("abc"), varchar("a"))); + Assertions.assertTrue(builtin.isBuiltin()); + for (Function.BinaryType udf : Arrays.asList(Function.BinaryType.JAVA_UDF, Function.BinaryType.PYTHON_UDF)) { + ConnectorFunctionCall converted = (ConnectorFunctionCall) ExprToConnectorExpressionConverter.convert( + call("starts_with", udf, varchar("abc"), varchar("a"))); + Assertions.assertFalse(converted.isBuiltin(), udf.name()); + // The name and the arguments are the builtin's, so the flag is all that tells the two apart. + Assertions.assertEquals(builtin.getFunctionName(), converted.getFunctionName()); + Assertions.assertEquals(builtin.getArguments(), converted.getArguments()); + Assertions.assertNotEquals(builtin, converted); + } + } + + @Test + void udfsNamedLikeOrRegexpStayFunctionCalls() { + for (String name : Arrays.asList("like", "regexp")) { + Assertions.assertInstanceOf(ConnectorLike.class, ExprToConnectorExpressionConverter.convert( + call(name, Function.BinaryType.BUILTIN, varchar("abc"), varchar("a%"))), name); + ConnectorExpression udf = ExprToConnectorExpressionConverter.convert( + call(name, Function.BinaryType.JAVA_UDF, varchar("abc"), varchar("a%"))); + Assertions.assertInstanceOf(ConnectorFunctionCall.class, udf, name); + Assertions.assertFalse(((ConnectorFunctionCall) udf).isBuiltin(), name); + } + } + + @Test + void fallbackIsBuiltinOnlyWhenTheExpressionItRendersCallsNoUdf() { + for (Function.BinaryType binaryType : Arrays.asList(Function.BinaryType.BUILTIN, + Function.BinaryType.JAVA_UDF)) { + // The converter does not model CASE, so the whole expression arrives as its SQL text. + Expr caseExpr = new CaseExpr(Collections.singletonList(new CaseWhenClause( + call("starts_with", binaryType, varchar("abc"), varchar("a")), integer(1))), integer(0), false); + ConnectorFunctionCall fallback = + (ConnectorFunctionCall) ExprToConnectorExpressionConverter.convert(caseExpr); + Assertions.assertTrue(fallback.getArguments().isEmpty()); + Assertions.assertTrue(fallback.getFunctionName().contains("starts_with"), fallback.getFunctionName()); + Assertions.assertEquals(binaryType == Function.BinaryType.BUILTIN, fallback.isBuiltin(), + binaryType.name()); + } + } +} diff --git a/regression-test/data/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.out b/regression-test/data/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.out new file mode 100644 index 00000000000000..f9c67531c91ff0 --- /dev/null +++ b/regression-test/data/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !udf_unique_name -- +112 + +-- !udf_builtin_name -- +112 + +-- !builtin -- +113 + diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.groovy new file mode 100644 index 00000000000000..5f75cc6c6cc36b --- /dev/null +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_mysql_udf_filter.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A Java UDF runs only in Doris, so a filter that calls one must not reach MySQL, even with function pushdown +// on: MySQL has no such function, or one of the same name that computes something else. +suite("test_jdbc_mysql_udf_filter", "p0,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String mysql_port = context.config.otherConfigs.get("mysql_57_port") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + + def jarPath = """${context.file.parent}/../../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" + scp_udf_file_to_all_be(jarPath) + scp_udf_file_to_all_fe(jarPath) + + // IntTest returns its argument plus one. It is registered under a name MySQL does not have and under + // abs, which MySQL and Doris both have; the database-qualified call reaches the UDF. + sql """drop database if exists test_jdbc_mysql_udf_filter_db""" + sql """create database test_jdbc_mysql_udf_filter_db""" + sql """use test_jdbc_mysql_udf_filter_db""" + sql """CREATE FUNCTION udf_add_one(int) RETURNS int PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.IntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + )""" + sql """CREATE FUNCTION abs(int) RETURNS int PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.IntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + )""" + + sql """drop catalog if exists test_jdbc_mysql_udf_filter_catalog""" + sql """create catalog test_jdbc_mysql_udf_filter_catalog properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver" + )""" + sql """set enable_ext_func_pred_pushdown = true""" + + // ex_tb0 holds the ids 111 to 115. + for (String udf : ["test_jdbc_mysql_udf_filter_db.udf_add_one", "test_jdbc_mysql_udf_filter_db.abs"]) { + explain { + sql("select id from test_jdbc_mysql_udf_filter_catalog.doris_test.ex_tb0 where ${udf}(id) = 113") + check { String plan -> + String remote = plan.readLines().find { it.contains("QUERY:") } + assertTrue(remote != null, plan) + assertFalse(remote.contains("WHERE"), remote) + } + } + } + order_qt_udf_unique_name """select id from test_jdbc_mysql_udf_filter_catalog.doris_test.ex_tb0 + where test_jdbc_mysql_udf_filter_db.udf_add_one(id) = 113""" + order_qt_udf_builtin_name """select id from test_jdbc_mysql_udf_filter_catalog.doris_test.ex_tb0 + where test_jdbc_mysql_udf_filter_db.abs(id) = 113""" + + // The builtin abs is still pushed. + explain { + sql("select id from test_jdbc_mysql_udf_filter_catalog.doris_test.ex_tb0 where abs(id) = 113") + check { String plan -> + String remote = plan.readLines().find { it.contains("QUERY:") } + assertTrue(remote != null && remote.contains("WHERE") && remote.contains("abs("), plan) + } + } + order_qt_builtin """select id from test_jdbc_mysql_udf_filter_catalog.doris_test.ex_tb0 where abs(id) = 113""" +}