Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,7 @@ public static EsQueryDslResult buildQueryDslWithResult(ConnectorExpression filte
}

private static boolean referencesUnsafeSourceField(ConnectorExpression expression, Set<String> unsafeFields) {
if (expression instanceof ConnectorFunctionCall
&& "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName())) {
if (isEsquery(expression)) {
// esquery explicitly requests ES index semantics rather than SQL source-value semantics.
return false;
}
Expand All @@ -194,8 +193,7 @@ private static boolean referencesUnsafeSourceField(ConnectorExpression expressio
}

private static void requireLocalEvaluation(ConnectorExpression expression) {
if (expression instanceof ConnectorFunctionCall
&& "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName())) {
if (isEsquery(expression)) {
// esquery has no BE implementation. Never leave it inside a residual OR/NOT that
// cannot be translated exactly; a remote match_all cannot make it locally evaluable.
throw new DorisConnectorException("Cannot evaluate esquery locally: its containing Boolean expression "
Expand Down Expand Up @@ -664,10 +662,16 @@ private static String convertLikeToWildcard(String pattern) {
// Function call (esquery)
// -----------------------------------------------------------------------

/** The builtin esquery(field, dsl), whose DSL Elasticsearch runs; a UDF of that name is not it. */
private static boolean isEsquery(ConnectorExpression expression) {
return expression instanceof ConnectorFunctionCall
&& ((ConnectorFunctionCall) expression).isBuiltin()
&& "esquery".equals(((ConnectorFunctionCall) expression).getFunctionName());
}

private static String functionCallToDsl(ConnectorFunctionCall funcCall,
List<ConnectorExpression> notPushDownList) {
if ("esquery".equals(funcCall.getFunctionName())
&& funcCall.getArguments().size() == 2) {
if (isEsquery(funcCall) && funcCall.getArguments().size() == 2) {
ConnectorExpression secondArg = funcCall.getArguments().get(1);
if (secondArg instanceof ConnectorLiteral) {
Object val = ((ConnectorLiteral) secondArg).getValue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.connector.es;

import org.apache.doris.connector.spi.ConnectorType;
import org.apache.doris.connector.spi.DorisConnectorException;
import org.apache.doris.connector.spi.pushdown.ConnectorAnd;
import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef;
import org.apache.doris.connector.spi.pushdown.ConnectorComparison;
Expand All @@ -33,6 +34,7 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class EsQueryDslBuilderTest {

Expand All @@ -59,6 +61,27 @@ void partialAndRetainsOnlyItsUnsupportedConjunct() {
Assertions.assertEquals("{\"term\":{\"name\":\"one\"}}", result.getQueryDsl());
}

@Test
void udfNamedEsqueryIsEvaluatedByDoris() {
List<ConnectorExpression> args = Arrays.asList(new ConnectorColumnRef("name", ConnectorType.of("VARCHAR")),
ConnectorLiteral.ofString("{\"term\":{\"name\":\"one\"}}"));
ConnectorExpression esquery = new ConnectorFunctionCall("esquery", ConnectorType.of("BOOLEAN"), args);
ConnectorExpression udf = new ConnectorFunctionCall("esquery", ConnectorType.of("BOOLEAN"), args, false);
EsQueryDslResult pushed = buildWithTracking(esquery);
Assertions.assertEquals("{\"term\":{\"name\":\"one\"}}", pushed.getQueryDsl());
Assertions.assertTrue(pushed.getNotPushedIndices().isEmpty());
EsQueryDslResult local = buildWithTracking(udf);
Assertions.assertEquals(EsQueryDslBuilder.matchAllQuery(), local.getQueryDsl());
Assertions.assertEquals(Collections.singletonList(0), local.getNotPushedIndices());
// In an OR that cannot be pushed, the builtin has no BE implementation to fall back to; the UDF has.
ConnectorExpression unpushable = new ConnectorFunctionCall("unsupported_predicate", ConnectorType.of("BOOLEAN"),
Collections.singletonList(new ConnectorColumnRef("name", ConnectorType.of("VARCHAR"))));
Assertions.assertThrows(DorisConnectorException.class,
() -> buildWithTracking(new ConnectorOr(Arrays.asList(esquery, unpushable))));
Assertions.assertEquals(Collections.singletonList(0),
buildWithTracking(new ConnectorOr(Arrays.asList(udf, unpushable))).getNotPushedIndices());
}

@Test
void orWithPartiallyPushableAndRemainsLocal() {
EsQueryDslResult result = buildWithTracking(new ConnectorOr(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,11 @@ private String betweenToSql(ConnectorBetween between, Map<String, String> colMap
private static final Set<String> UNARY_OPERATORS = Collections.singleton("~");

private String functionCallToSql(ConnectorFunctionCall func, Map<String, String> colMapping) {
// A UDF runs only in Doris: the remote database has no such function, or one of the same name that
// computes something else. This also keeps a pre-rendered fragment that calls one local.
if (!func.isBuiltin()) {
return null;
}
String funcName = func.getFunctionName();

// 0. Handle arithmetic operators rendered as infix: (arg1 + arg2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,29 @@ void testPreRenderedOperandIsGrouped() {
ConnectorComparison.Operator.EQ, fragment, ConnectorLiteral.ofBoolean(false))), -1));
}

@Test
void testUserDefinedFunctionsRemainLocalWithoutLimit() {
JdbcFunctionPushdownConfig config = JdbcFunctionPushdownConfig.create(JdbcDbType.MYSQL, null, true);
JdbcQueryBuilder builder = new JdbcQueryBuilder(JdbcDbType.MYSQL, config, Collections.emptyMap());
List<ConnectorExpression> args = Collections.singletonList(new ConnectorColumnRef("id", INT_TYPE));
// A UDF named like MySQL's abs, and a pre-rendered fragment that calls a UDF.
for (ConnectorExpression udf : Arrays.asList(new ConnectorFunctionCall("abs", INT_TYPE, args, false),
new ConnectorFunctionCall("CASE WHEN my_udf(`id`) THEN 1 END", INT_TYPE,
Collections.emptyList(), false))) {
ConnectorExpression predicate = new ConnectorComparison(
ConnectorComparison.Operator.EQ, udf, ConnectorLiteral.ofInt(1));
Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl`",
builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of(predicate), 1));
Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl` WHERE (`id` = 1)",
builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of(
new ConnectorAnd(Arrays.asList(predicate, simpleComparison("id", 1)))), 1));
}
Assertions.assertEquals("SELECT * FROM `testdb`.`testtbl` WHERE ((abs(`id`)) = 1) LIMIT 1",
builder.buildQuery(DB, TABLE, Collections.emptyList(), Optional.of(new ConnectorComparison(
ConnectorComparison.Operator.EQ, new ConnectorFunctionCall("abs", INT_TYPE, args),
ConnectorLiteral.ofInt(1))), 1));
}

@Test
void testEmptyInRemainsLocalWithoutLimit() {
ConnectorColumnRef column = new ConnectorColumnRef("id", INT_TYPE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
/**
* A function call expression (catch-all for unsupported function types).
* Connector plugins can inspect the function name and arguments to decide
* whether to push down or reject.
* whether to push down or reject; only a call that {@link #isBuiltin() is a builtin} may be matched by name.
*/
public final class ConnectorFunctionCall implements ConnectorExpression {

Expand All @@ -35,21 +35,41 @@ public final class ConnectorFunctionCall implements ConnectorExpression {
private final String functionName;
private final ConnectorType returnType;
private final List<ConnectorExpression> arguments;
private final boolean builtin;

/** A call of a Doris builtin function or operator. */
public ConnectorFunctionCall(String functionName,
ConnectorType returnType,
List<ConnectorExpression> arguments) {
this(functionName, returnType, arguments, true);
}

public ConnectorFunctionCall(String functionName,
ConnectorType returnType,
List<ConnectorExpression> arguments,
boolean builtin) {
this.functionName = Objects.requireNonNull(functionName, "functionName");
this.returnType = Objects.requireNonNull(returnType, "returnType");
this.arguments = arguments == null
? Collections.emptyList()
: Collections.unmodifiableList(arguments);
this.builtin = builtin;
}

public String getFunctionName() {
return functionName;
}

/**
* Whether this calls a Doris builtin function or operator. A user-defined function (a Java or Python UDF)
* does not: its name says nothing about what it computes, so a connector must never translate it by name.
* For the fallback carrier (Rule 6 of the {@code pushdown} package), false when the expression it renders
* calls a user-defined function.
*/
public boolean isBuiltin() {
return builtin;
}

public ConnectorType getReturnType() {
return returnType;
}
Expand Down Expand Up @@ -79,11 +99,12 @@ public boolean equals(Object o) {
ConnectorFunctionCall that = (ConnectorFunctionCall) o;
return functionName.equals(that.functionName)
&& returnType.equals(that.returnType)
&& arguments.equals(that.arguments);
&& arguments.equals(that.arguments)
&& builtin == that.builtin;
}

@Override
public int hashCode() {
return Objects.hash(functionName, returnType, arguments);
return Objects.hash(functionName, returnType, arguments, builtin);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -152,14 +152,22 @@
* original expression as the residual, so their predicates are also evaluated on BE. That is correct but not
* free — and it is the reason a slightly-too-wide pushdown is survivable today.</p>
*
* <h2>Rule 6 — {@link ConnectorFunctionCall} is also the fallback carrier. Do not match it by name</h2>
*
* <p>{@code ExprToConnectorExpressionConverter.fallback} turns any expression it does not model into a
* {@link ConnectorFunctionCall} whose <b>name is the rendered Doris SQL text</b> and whose argument list is
* <b>empty</b>. It is not a function call at all. Matching function names blindly will make a connector
* translate {@code my_udf(a, b) = 1} as a call to a function literally named {@code "my_udf(a, b) = 1"}.
* jdbc handles this deliberately: an argument-less call whose name is not a plain identifier is emitted as a
* pre-rendered SQL fragment ({@code JdbcQueryBuilder}).</p>
* <h2>Rule 6 — {@link ConnectorFunctionCall} is also the fallback carrier and the user-defined function.
* Match it by name only when it is a builtin</h2>
*
* <p>{@code ExprToConnectorExpressionConverter.fallback} turns any expression it does not model, such as a
* {@code CASE} expression, into a {@link ConnectorFunctionCall} whose <b>name is the rendered Doris SQL text</b>
* and whose argument list is <b>empty</b>. It is not a function call at all. Matching function names blindly
* will make a connector translate {@code CASE WHEN a > 0 THEN b END} as a call to a function literally named
* by that text. jdbc handles this deliberately: an argument-less call whose name is not a plain identifier is
* emitted as a pre-rendered SQL fragment ({@code JdbcQueryBuilder}).</p>
*
* <p>A user-defined function (a Java or Python UDF) arrives with its own name and real arguments, exactly like
* a builtin, but with {@link ConnectorFunctionCall#isBuiltin()} false
* ({@code ExprToConnectorExpressionConverter.convertFunctionCall}). Its name says nothing about what it
* computes: a UDF may be named {@code starts_with} or {@code esquery}, and no remote system runs it. Never
* translate such a call. A fallback whose rendered expression calls one is not a builtin either, so a
* connector that emits fallbacks as SQL must check it too.</p>
*
* <p>Two related shapes to expect:</p>
* <ul>
Expand All @@ -168,8 +176,8 @@
* {@code ConnectorFunctionCall} named {@code like} with three arguments. A connector that only handles
* {@link ConnectorLike} therefore never sees a custom escape character, and may assume the fixed
* backslash escape documented on {@link ConnectorLike}.</li>
* <li>Arithmetic and genuine scalar functions arrive as {@link ConnectorFunctionCall} with real arguments;
* those are safe to match by name.</li>
* <li>Arithmetic and builtin scalar functions arrive as {@link ConnectorFunctionCall} with real arguments and
* {@link ConnectorFunctionCall#isBuiltin()} true; those are safe to match by name.</li>
* </ul>
*
* <p>Anything you cannot interpret confidently falls under Rule 1: drop the conjunct.</p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException
Assertions.assertNotNull(in, "missing connector plugin API version resource");
version.load(in);
}
// Major 29 adds typed table-absence errors after the upstream API 28 changes.
Assertions.assertEquals("30.0", version.getProperty("api.version"));
// Major 31 adds ConnectorFunctionCall#isBuiltin, which tells a UDF from a builtin of the same name.
Assertions.assertEquals("31.0", version.getProperty("api.version"));
}

/** Root entry points plus provider/handle types returned to connector plugins. */
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.connector.spi.pushdown;

import org.apache.doris.connector.spi.ConnectorType;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.Collections;
import java.util.List;

public class ConnectorFunctionCallTest {
private static final ConnectorType BOOLEAN = ConnectorType.of("BOOLEAN");
private static final List<ConnectorExpression> ARGS = Collections.singletonList(ConnectorLiteral.ofString("a"));

@Test
public void callIsBuiltinUnlessCreatedAsUdf() {
Assertions.assertTrue(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS).isBuiltin());
Assertions.assertTrue(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, true).isBuiltin());
Assertions.assertFalse(new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, false).isBuiltin());
}

@Test
public void udfDiffersFromTheBuiltinOfTheSameName() {
ConnectorFunctionCall builtin = new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS);
ConnectorFunctionCall alsoBuiltin = new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, true);
Assertions.assertEquals(builtin, alsoBuiltin);
Assertions.assertEquals(builtin.hashCode(), alsoBuiltin.hashCode());
Assertions.assertNotEquals(builtin, new ConnectorFunctionCall("starts_with", BOOLEAN, ARGS, false));
}
}
2 changes: 1 addition & 1 deletion fe/fe-connector/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ under the License.
of the latter two means bumping this property as well (and fe-extension-spi means bumping
all five families).
-->
<connector.plugin.api.version>30.0</connector.plugin.api.version>
<connector.plugin.api.version>31.0</connector.plugin.api.version>
</properties>

<modules>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructField;
Expand Down Expand Up @@ -114,10 +115,10 @@ public static ConnectorExpression convert(Expr expr) {
String fnName = fce.getFnName().getFunction();
// Nereids translates Like/Regexp to FunctionCallExpr via visitScalarFunction,
// so we need to detect them here and convert to ConnectorLike.
if ("like".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) {
if (isBuiltin(fce) && "like".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) {
return new ConnectorLike(ConnectorLike.Operator.LIKE,
convert(fce.getChild(0)), convert(fce.getChild(1)));
} else if ("regexp".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) {
} else if (isBuiltin(fce) && "regexp".equalsIgnoreCase(fnName) && fce.getChildren().size() == 2) {
return new ConnectorLike(ConnectorLike.Operator.REGEXP,
convert(fce.getChild(0)), convert(fce.getChild(1)));
}
Expand Down Expand Up @@ -336,7 +337,12 @@ private static ConnectorExpression convertFunctionCall(FunctionCallExpr fce) {
List<ConnectorExpression> args = fce.getChildren().stream()
.map(ExprToConnectorExpressionConverter::convert)
.collect(Collectors.toList());
return new ConnectorFunctionCall(fnName, retType, args);
// A UDF keeps only its bare name here, which may be a builtin's; the flag is what tells them apart.
return new ConnectorFunctionCall(fnName, retType, args, isBuiltin(fce));
}

private static boolean isBuiltin(FunctionCallExpr fce) {
return fce.getFn().getBinaryType() == Function.BinaryType.BUILTIN;
}

private static ConnectorExpression fallback(Expr expr) {
Expand All @@ -351,7 +357,9 @@ private static ConnectorExpression fallback(Expr expr) {
} catch (Exception e) {
sql = expr.debugString();
}
return new ConnectorFunctionCall(sql, retType, Collections.emptyList());
boolean callsUdf = expr.contains(
(Expr node) -> node instanceof FunctionCallExpr && !isBuiltin((FunctionCallExpr) node));
return new ConnectorFunctionCall(sql, retType, Collections.emptyList(), !callsUdf);
}

private static ConnectorType scalarTypeToConnectorType(ScalarType st) {
Expand Down
Loading
Loading