From 3a94e224f758df5be14bd1d08bb5b546cca5b319 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 14:18:56 +0800 Subject: [PATCH 1/7] [fix](external) Preserve source values and alias semantics in ES scans (#69672) Adapt mapping safety, predicate residuals and alias routing to the legacy ES planner on branch-4.2. (cherry picked from commit c7d1df40c00914ed9dffcdba61aada94b5e5b3cd) --- be/src/exec/es/es_scroll_parser.cpp | 480 +++- be/src/exec/es/es_scroll_query.cpp | 5 + be/test/exec/es_scroll_parser_test.cpp | 481 ++++ .../org/apache/doris/catalog/EsTable.java | 6 + .../doris/datasource/es/EsFieldContext.java | 95 + .../doris/datasource/es/EsMappingUtils.java | 441 +++ .../doris/datasource/es/EsRestClient.java | 3 +- .../apache/doris/datasource/es/EsUtil.java | 85 +- .../doris/datasource/es/MappingPhase.java | 128 +- .../doris/datasource/es/QueryBuilders.java | 91 +- .../doris/datasource/es/SearchContext.java | 41 +- .../datasource/es/source/EsScanNode.java | 69 +- .../datasource/es/EsMappingUtilsTest.java | 225 ++ .../es/EsRestClientVersionTest.java | 26 + .../source/EsScanNodeFileScannerV2Test.java | 111 + .../elasticsearch/QueryBuildersTest.java | 27 +- .../es/test_es_epoch_millis.out | 721 +++++ .../es/test_es_mapping_boundaries.out | 1129 ++++++++ .../es/test_es_numeric_source_semantics.out | 813 ++++++ .../es/test_es_scalar_materialization.out | 989 +++++++ .../es/test_es_source_fallback_types.out | 1141 ++++++++ .../es/test_es_source_semantics.out | 2509 +++++++++++++++++ .../es/test_es_source_value_boundaries.out | 649 +++++ .../es/test_es_epoch_millis.groovy | 113 + .../es/test_es_mapping_boundaries.groovy | 141 + .../test_es_numeric_source_semantics.groovy | 141 + .../es/test_es_scalar_materialization.groovy | 156 + .../es/test_es_source_fallback_types.groovy | 230 ++ .../es/test_es_source_semantics.groovy | 339 +++ .../es/test_es_source_value_boundaries.groovy | 156 + 30 files changed, 11173 insertions(+), 368 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java create mode 100644 regression-test/data/external_table_p2/es/test_es_epoch_millis.out create mode 100644 regression-test/data/external_table_p2/es/test_es_mapping_boundaries.out create mode 100644 regression-test/data/external_table_p2/es/test_es_numeric_source_semantics.out create mode 100644 regression-test/data/external_table_p2/es/test_es_scalar_materialization.out create mode 100644 regression-test/data/external_table_p2/es/test_es_source_fallback_types.out create mode 100644 regression-test/data/external_table_p2/es/test_es_source_semantics.out create mode 100644 regression-test/data/external_table_p2/es/test_es_source_value_boundaries.out create mode 100644 regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy create mode 100644 regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 5456c0bfa7172f..dd10d5eb53f67f 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -27,9 +27,12 @@ // IWYU pragma: no_include #include // IWYU pragma: keep +#include #include +#include #include #include +#include #include "common/status.h" #include "core/binary_cast.hpp" @@ -126,17 +129,6 @@ static const std::string INVALID_NULL_VALUE = } \ } while (false) -#define RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col, type) \ - do { \ - if (!col.IsNumber()) { \ - std::stringstream ss; \ - ss << "Expected value of type: " << type_to_string(type) \ - << "; but found type: " << json_type_to_string(col.GetType()) \ - << "; Document value is: " << json_value_to_string(col); \ - return Status::RuntimeError(ss.str()); \ - } \ - } while (false) - #define RETURN_ERROR_IF_PARSING_FAILED(result, col, type) \ do { \ if (result != StringParser::PARSE_SUCCESS) { \ @@ -157,39 +149,285 @@ static const std::string INVALID_NULL_VALUE = return Status::RuntimeError(ss.str()); \ } while (false) +// ES integer coercion truncates decimal/scientific notation, e.g. "1.2e3" -> 1200. +// Shift the decimal point using digits so values near INT64_MAX never pass through double. template -Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void* slot, - bool pure_doc_value) { - if (col.IsNumber()) { - *reinterpret_cast(slot) = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64()); - return Status::OK(); +static T parse_es_integer(const char* text, size_t length, StringParser::ParseResult* result) { + text = skip_ascii_whitespaces(text, length); + size_t i = 0; + bool negative = length > 0 && text[0] == '-'; + if (length > 0 && (text[0] == '-' || text[0] == '+')) { + ++i; } - - if (pure_doc_value && col.IsArray() && !col.Empty()) { - RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type); - *reinterpret_cast(slot) = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64()); - return Status::OK(); + size_t start = i; + int64_t digits = 0; + int64_t integer_digits = 0; + int64_t first_nonzero = -1; + bool dot = false; + for (; i < length; ++i) { + char c = text[i]; + if (c >= '0' && c <= '9') { + if (c != '0' && first_nonzero < 0) { + first_nonzero = digits; + } + ++digits; + integer_digits += !dot; + } else if (c == '.' && !dot) { + dot = true; + } else { + break; + } + } + size_t end = i; + int64_t exponent = 0; + bool negative_exponent = false; + if (i < length && (text[i] == 'e' || text[i] == 'E')) { + ++i; + if (i < length && (text[i] == '+' || text[i] == '-')) { + negative_exponent = text[i++] == '-'; + } + size_t exponent_start = i; + for (; i < length && text[i] >= '0' && text[i] <= '9'; ++i) { + // Saturation avoids overflow and work proportional to an untrusted exponent. + exponent = std::min(exponent * 10 + text[i] - '0', static_cast(length) + 64); + } + if (i == exponent_start) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + } + if (digits == 0 || i != length) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + int64_t count = integer_digits + (negative_exponent ? -exponent : exponent) - first_nonzero; + if (first_nonzero < 0 || count <= 0) { + *result = StringParser::PARSE_SUCCESS; + return 0; + } + // 39 digits cover every signed 128-bit value; the final parser checks the exact type bounds. + if (count > 39) { + *result = StringParser::PARSE_OVERFLOW; + return 0; + } + char integral[40]; + size_t size = 0; + if (negative) { + integral[size++] = '-'; + } + int64_t digit = 0; + for (i = start; i < end && count > 0; ++i) { + if (text[i] != '.' && digit++ >= first_nonzero) { + integral[size++] = text[i]; + --count; + } } + while (count-- > 0) { + integral[size++] = '0'; + } + return StringParser::string_to_int(integral, size, result); +} - RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true); - RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); +template +static T parse_es_floating_string(const std::string& text, StringParser::ParseResult* result) { + // Java's floating parser also accepts hexadecimal exponents, f/d suffixes and ASCII trim + // characters. Keep the ordinary decimal fast path, and require the complete Java token. + std::string_view token(text); + while (!token.empty() && static_cast(token.front()) <= 0x20) { + token.remove_prefix(1); + } + while (!token.empty() && static_cast(token.back()) <= 0x20) { + token.remove_suffix(1); + } + auto magnitude = token; + if (!magnitude.empty() && (magnitude.front() == '+' || magnitude.front() == '-')) { + magnitude.remove_prefix(1); + } + if (magnitude == "NaN" || magnitude == "Infinity") { + return StringParser::string_to_float(token.data(), token.size(), result); + } + if (!magnitude.empty() && + std::string_view("fFdD").find(magnitude.back()) != std::string_view::npos) { + token.remove_suffix(1); + magnitude.remove_suffix(1); + // Java permits outer whitespace, but not a gap between the number and its suffix. + if (magnitude.empty() || static_cast(magnitude.back()) <= 0x20) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + } + if (magnitude.empty() || + !((magnitude.front() >= '0' && magnitude.front() <= '9') || magnitude.front() == '.')) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + const bool hexadecimal = magnitude.starts_with("0x") || magnitude.starts_with("0X"); + if (hexadecimal && magnitude.find_first_of("pP") == std::string_view::npos) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + // Float.parseFloat rounds directly to float; an intermediate double can round a value + // just above a float midpoint back to that midpoint and then to the wrong float. + if (hexadecimal || std::is_same_v) { + char* end = nullptr; + T number; + if constexpr (std::is_same_v) { + number = std::strtof(token.data(), &end); + } else { + number = std::strtod(token.data(), &end); + } + *result = end == token.data() + token.size() ? StringParser::PARSE_SUCCESS + : StringParser::PARSE_FAILURE; + return number; + } + return StringParser::string_to_float(token.data(), token.size(), result); +} - StringParser::ParseResult result; - const std::string& val = col.GetString(); - size_t len = col.GetStringLength(); - T v = StringParser::string_to_int(val.c_str(), len, &result); - RETURN_ERROR_IF_PARSING_FAILED(result, col, type); +template +static T parse_es_narrow_integer(const std::string& text, StringParser::ParseResult* result) { + // Unlike long's BigDecimal conversion, ES parses byte/short/integer strings through double: + // "0.99999999999999999" rounds to 1 before truncation. Keep long's exact path separate. + auto number = parse_es_floating_string(text, result); + if (*result != StringParser::PARSE_SUCCESS) { + return 0; + } + // Java converts NaN to zero; casting it directly in C++ would be undefined. + if (std::isnan(number)) { + return 0; + } + using ParsedType = std::conditional_t; + if (number < StringParser::numeric_limits(true) || + number > StringParser::numeric_limits(false)) { + *result = StringParser::PARSE_OVERFLOW; + return 0; + } + auto parsed = static_cast(number); + // ES byte uses intValue() then checks the byte range, so "127.9" is valid. Short and + // integer instead check their own double bounds before casting (e.g. short "32767.9" fails). + if constexpr (sizeof(T) == 1) { + if (parsed < StringParser::numeric_limits(true) || + parsed > StringParser::numeric_limits(false)) { + *result = StringParser::PARSE_OVERFLOW; + return 0; + } + } + return static_cast(parsed); +} - if (sizeof(T) < 16) { - *reinterpret_cast(slot) = v; +template +Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void* slot, + bool pure_doc_value) { + const rapidjson::Value* value = &col; + // A singleton source array has the same scalar value as its one docvalue. Keep rejecting + // multi-value source arrays: selecting an arbitrary element would discard source semantics. + if (col.IsArray() && !col.Empty() && (pure_doc_value || col.Size() == 1)) { + value = &col[0]; + } + RETURN_ERROR_IF_COL_IS_ARRAY((*value), type, true); + T parsed; + if (value->IsInt64() || value->IsUint64()) { + __int128 number = value->IsInt64() ? static_cast<__int128>(value->GetInt64()) + : static_cast<__int128>(value->GetUint64()); + if (number < StringParser::numeric_limits(true) || + number > StringParser::numeric_limits(false)) { + return Status::RuntimeError("ES integer value is out of range for {}", + type_to_string(type)); + } + parsed = static_cast(number); + } else if (value->IsNumber()) { + // Jackson converts numeric tokens directly from double. Serializing INT64_MIN first + // rounds its text below the valid range; Java also saturates the rounded upper endpoint. + const double number = value->GetDouble(); + using ParsedType = std::conditional_t<(sizeof(T) < 4), int32_t, T>; + const auto maximum = static_cast(StringParser::numeric_limits(false)); + if (!std::isfinite(number) || + number < static_cast(StringParser::numeric_limits(true)) || + number > maximum) { + return Status::RuntimeError("ES integer value is out of range for {}", + type_to_string(type)); + } + const ParsedType converted = number == maximum + ? StringParser::numeric_limits(false) + : static_cast(number); + if constexpr (sizeof(T) < 4) { + if (converted < StringParser::numeric_limits(true) || + converted > StringParser::numeric_limits(false)) { + return Status::RuntimeError("ES integer value is out of range for {}", + type_to_string(type)); + } + } + parsed = static_cast(converted); } else { - DCHECK(sizeof(T) == 16); - memcpy(slot, &v, sizeof(v)); + RETURN_ERROR_IF_COL_IS_NOT_STRING((*value), type); + std::string text(value->GetString(), value->GetStringLength()); + StringParser::ParseResult result; + if constexpr (sizeof(T) < 8) { + parsed = parse_es_narrow_integer(text, &result); + } else { + parsed = parse_es_integer(text.data(), text.size(), &result); + } + RETURN_ERROR_IF_PARSING_FAILED(result, (*value), type); } - + memcpy(slot, &parsed, sizeof(parsed)); return Status::OK(); } +// ES epoch_millis accepts signed decimal milliseconds with up to six fractional digits. +// Parse the integer part exactly: a double round trip can move a long timestamp across a second. +static bool parse_es_epoch_millis(std::string_view text, int64_t* seconds) { + bool negative = !text.empty() && text.front() == '-'; + size_t pos = negative ? 1 : 0; + size_t start = pos; + while (pos < text.size() && text[pos] >= '0' && text[pos] <= '9') { + ++pos; + } + size_t integer_end = pos; + // ES rejects empty numbers and integer parts wider than its signed-long grammar. + if (pos == start || pos - start > 19) { + return false; + } + bool nonzero_fraction = false; + if (pos < text.size() && text[pos] == '.') { + start = ++pos; + while (pos < text.size() && text[pos] >= '0' && text[pos] <= '9') { + nonzero_fraction |= text[pos++] != '0'; + } + // epoch_millis fractions express nanoseconds, so a seventh digit is invalid. + if (pos - start > 6) { + return false; + } + } + // Reject exponents, suffixes and trailing junk instead of accepting a numeric prefix. + if (pos != text.size()) { + return false; + } + StringParser::ParseResult result; + auto millis = StringParser::string_to_int(text.data(), integer_end, &result); + // Out-of-range milliseconds must not wrap into an otherwise valid calendar date. + if (result != StringParser::PARSE_SUCCESS) { + return false; + } + *seconds = millis / 1000; + // At second precision, -1000.5 ms belongs to 23:59:58, not 23:59:59. + if (negative && (millis % 1000 != 0 || nonzero_fraction)) { + --*seconds; + } + return true; +} + +template +static bool set_es_epoch_seconds(DateValue& value, int64_t seconds, + const cctz::time_zone& time_zone) { + const auto civil = + cctz::convert(cctz::time_point(cctz::seconds(seconds)), time_zone); + // from_unixtime narrows the civil year without checking it; e.g. year 10000 must fail first. + if (civil.year() < 0 || civil.year() > MAX_YEAR) { + return false; + } + value.from_unixtime(seconds, time_zone); + return true; +} + template Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool is_date_str, typename PrimitiveTypeTraits::CppType* slot, @@ -197,10 +435,20 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME; typename PrimitiveTypeTraits::CppType dt_val; if (is_date_str) { - const std::string str_date = col.GetString(); - int str_length = col.GetStringLength(); + std::string str_date(col.GetString(), col.GetStringLength()); + size_t str_length = col.GetStringLength(); + // The default strict_date_optional_time parser wins over epoch_millis for a four-digit year. + // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds after the epoch. + if (str_length == 4 && str_date.find_first_not_of("0123456789") == std::string::npos) { + str_date += "-01-01"; + str_length = str_date.size(); + } bool success = false; - if (str_length > 19) { + int64_t epoch_seconds; + // Source fallback also reaches short, zero and fractional epoch strings, e.g. "0"/"1.5". + if (parse_es_epoch_millis(str_date, &epoch_seconds)) { + success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone); + } else if (str_length > 19) { std::chrono::system_clock::time_point tp; // time_zone suffix pattern // Z/+08:00/-04:30 @@ -229,7 +477,10 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool // The local time zone can change by session variable `time_zone` // We should use the user specified time zone, not the actual system local time zone. success = true; - dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone); + // A pre-epoch fractional instant must floor to its containing second, like numeric dates. + dt_val.from_unixtime( + std::chrono::floor(tp).time_since_epoch().count(), + time_zone); } } else if (str_length == 19) { // YYYY-MM-DDTHH:MM:SS @@ -246,37 +497,30 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool CastParameters params; if constexpr (is_datetime_v1) { success = CastToDateOrDatetime::from_string_non_strict_mode< - DatelikeTargetType::DATE_TIME>({str_date.c_str(), (size_t)str_length}, - dt_val, nullptr, params); + DatelikeTargetType::DATE_TIME>({str_date.c_str(), str_length}, dt_val, + nullptr, params); } else if constexpr (T == TYPE_DATEV2) { success = CastToDateV2::from_string_non_strict_mode( - {str_date.c_str(), (size_t)str_length}, dt_val, nullptr, params); + {str_date.c_str(), str_length}, dt_val, nullptr, params); } else { success = CastToDatetimeV2::from_string_non_strict_mode( - {str_date.c_str(), (size_t)str_length}, dt_val, nullptr, -1, params); + {str_date.c_str(), str_length}, dt_val, nullptr, -1, params); } } - } else if (str_length == 13) { - // string long like "1677895728000" - int64_t time_long = std::atol(str_date.c_str()); - if (time_long > 0) { - success = true; - dt_val.from_unixtime(time_long / 1000, time_zone); - } } else { // YYYY-MM-DD or others CastParameters params; if constexpr (is_datetime_v1) { success = CastToDateOrDatetime::from_string_non_strict_mode< - DatelikeTargetType::DATE_TIME>({str_date.c_str(), (size_t)str_length}, - dt_val, nullptr, params); + DatelikeTargetType::DATE_TIME>({str_date.c_str(), str_length}, dt_val, + nullptr, params); } else if constexpr (T == TYPE_DATEV2) { - success = CastToDateV2::from_string_non_strict_mode( - {str_date.c_str(), (size_t)str_length}, dt_val, nullptr, params); + success = CastToDateV2::from_string_non_strict_mode({str_date.c_str(), str_length}, + dt_val, nullptr, params); } else { success = CastToDatetimeV2::from_string_non_strict_mode( - {str_date.c_str(), (size_t)str_length}, dt_val, nullptr, -1, params); + {str_date.c_str(), str_length}, dt_val, nullptr, -1, params); } } @@ -285,7 +529,26 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool } } else { - dt_val.from_unixtime(col.GetInt64() / 1000, time_zone); + int64_t epoch_seconds; + // Integral JSON values retain their exact bits; unsigned overflow cannot be a Doris date. + if (col.IsInt64()) { + int64_t millis = col.GetInt64(); + epoch_seconds = millis / 1000 - (millis % 1000 < 0); + } else if (col.IsDouble()) { + double seconds = std::floor(col.GetDouble() / 1000); + constexpr double minimum = std::numeric_limits::min(); + // Fractional numeric dates use GetDouble; check before casting NaN/overflow to int64. + if (!(seconds >= minimum && seconds < -minimum)) { + RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); + } + epoch_seconds = static_cast(seconds); + } else { + RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); + } + // Reject civil-year overflow before from_unixtime narrows it into the date representation. + if (!set_es_epoch_seconds(dt_val, epoch_seconds, time_zone)) { + RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); + } } if constexpr (is_datetime_v1) { if (type == TYPE_DATE) { @@ -354,9 +617,8 @@ Status get_float_value(const rapidjson::Value& col, PrimitiveType type, void* sl RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); StringParser::ParseResult result; - const std::string& val = col.GetString(); - size_t len = col.GetStringLength(); - T v = StringParser::string_to_float(val.c_str(), len, &result); + const std::string val(col.GetString(), col.GetStringLength()); + T v = parse_es_floating_string(val, &result); RETURN_ERROR_IF_PARSING_FAILED(result, col, type); *reinterpret_cast(slot) = v; @@ -383,9 +645,8 @@ Status insert_float_value(const rapidjson::Value& col, PrimitiveType type, IColu RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); StringParser::ParseResult result; - const std::string& val = col.GetString(); - size_t len = col.GetStringLength(); - T v = StringParser::string_to_float(val.c_str(), len, &result); + const std::string val(col.GetString(), col.GetStringLength()); + T v = parse_es_floating_string(val, &result); RETURN_ERROR_IF_PARSING_FAILED(result, col, type); col_ptr->insert_data(const_cast(reinterpret_cast(&v)), 0); @@ -396,53 +657,24 @@ Status insert_float_value(const rapidjson::Value& col, PrimitiveType type, IColu template Status insert_int_value(const rapidjson::Value& col, PrimitiveType type, IColumn* col_ptr, bool pure_doc_value, bool nullable) { - if (col.IsNumber()) { - T value; - // ES allows inserting float and double in int/long types. - // To parse these numbers in Doris, we direct cast them to int types. - if (col.IsDouble()) { - value = static_cast(col.GetDouble()); - } else if (col.IsFloat()) { - value = static_cast(col.GetFloat()); - } else { - value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64()); - } - col_ptr->insert_data(const_cast(reinterpret_cast(&value)), 0); + // Empty arrays and singleton nulls have no scalar integer value, just like absent docvalues. + if (nullable && col.IsArray() && (col.Empty() || (col.Size() == 1 && col[0].IsNull()))) { + col_ptr->insert_data(nullptr, 0); return Status::OK(); } + T value; + RETURN_IF_ERROR(get_int_value(col, type, &value, pure_doc_value)); + col_ptr->insert_data(reinterpret_cast(&value), 0); + return Status::OK(); +} - auto parse_and_insert_data = [&](const rapidjson::Value& col_value) -> Status { - StringParser::ParseResult result; - std::string val = col_value.GetString(); - // ES allows inserting numbers and characters containing decimals in numeric types. - // To parse these numbers in Doris, we remove the decimals here. - size_t pos = val.find('.'); - if (pos != std::string::npos) { - val = val.substr(0, pos); - } - size_t len = val.length(); - T v = StringParser::string_to_int(val.c_str(), len, &result); - RETURN_ERROR_IF_PARSING_FAILED(result, col_value, type); - - col_ptr->insert_data(const_cast(reinterpret_cast(&v)), 0); - return Status::OK(); - }; - - if (pure_doc_value && col.IsArray() && !col.Empty()) { - if (col[0].IsNumber()) { - T value = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64()); - col_ptr->insert_data(const_cast(reinterpret_cast(&value)), 0); - return Status::OK(); - } else { - RETURN_ERROR_IF_COL_IS_ARRAY(col[0], type, true); - RETURN_ERROR_IF_COL_IS_NOT_STRING(col[0], type); - return parse_and_insert_data(col[0]); - } +static bool parse_es_bool(const char* value, size_t length, StringParser::ParseResult* result) { + // ES accepts an empty string as false in source, for scalar fields and array elements alike. + if (length == 0) { + *result = StringParser::PARSE_SUCCESS; + return false; } - - RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true); - RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); - return parse_and_insert_data(col); + return StringParser::string_to_bool(value, length, result); } template @@ -500,7 +732,7 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu const std::string& str_val = str_col.GetString(); size_t val_size = str_col.GetStringLength(); StringParser::ParseResult result; - val = StringParser::string_to_bool(str_val.c_str(), val_size, &result); + val = parse_es_bool(str_val.c_str(), val_size, &result); RETURN_ERROR_IF_PARSING_FAILED(result, str_col, sub_type); return Status::OK(); } @@ -510,6 +742,13 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu template Status process_single_column(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value, Array& array) { + // Source arrays retain null members that ES omits from docvalues. Numeric empty strings + // have the same missing-value semantics; Boolean empty strings still mean false. + if (col.IsNull() || + (is_number(T) && T != TYPE_BOOLEAN && col.IsString() && col.GetStringLength() == 0)) { + array.push_back(Field {}); + return Status::OK(); + } typename PrimitiveTypeTraits::CppType val; RETURN_IF_ERROR(handle_value(col, sub_type, pure_doc_value, val)); array.push_back(Field::create_field(val)); @@ -544,6 +783,11 @@ Status process_date_column(const rapidjson::Value& col, PrimitiveType sub_type, array.push_back(Field::create_field(data)); } else { for (const auto& sub_col : col.GetArray()) { + // Nullable source members must bypass date parsing, which requires a real value. + if (sub_col.IsNull()) { + array.push_back(Field {}); + continue; + } typename PrimitiveTypeTraits::CppType data; RETURN_IF_ERROR((get_date_int(sub_col, sub_type, pure_doc_value, &data, time_zone))); array.push_back(Field::create_field(data)); @@ -561,6 +805,11 @@ Status process_jsonb_column(const rapidjson::Value& col, PrimitiveType sub_type, array.push_back(Field::create_field(std::move(json))); } else { for (const auto& sub_col : col.GetArray()) { + // SQL NULL members stay distinct from nonnull JSONB containers such as [null]. + if (sub_col.IsNull()) { + array.push_back(Field {}); + continue; + } JsonBinaryValue jsonb_value; RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(sub_col))); JsonbField json(jsonb_value.value(), jsonb_value.size()); @@ -724,15 +973,24 @@ Status ScrollParser::fill_columns(const std::vector& slots, } } - const rapidjson::Value& col = (*line)[col_name]; - auto type = slot_desc->type()->get_primitive_type(); - - // when the column value is null, the subsequent type casting will report an error - if (col.IsNull() && slot_desc->is_nullable()) { + const rapidjson::Value& source_value = (*line)[col_name]; + // Source fallback changes every sibling's decoder. A scalar numeric/Boolean/date field + // may hold [value], [] or [null]; preserve its single value or missing-value semantics. + // STRING and JSONB keep their JSON representation; declared ARRAY keeps its container. + bool scalar_source_array = !pure_doc_value && source_value.IsArray() && + type != TYPE_ARRAY && type != TYPE_JSONB && + !is_string_type(type); + const rapidjson::Value& col = + scalar_source_array && source_value.Size() == 1 ? source_value[0] : source_value; + // Empty numeric source strings are ES missing values, including singleton arrays. + bool is_null = col.IsNull() || (scalar_source_array && source_value.Empty()) || + (!pure_doc_value && is_number(type) && type != TYPE_BOOLEAN && + col.IsString() && col.GetStringLength() == 0); + if (is_null && slot_desc->is_nullable()) { col_ptr->insert_data(nullptr, 0); continue; - } else if (col.IsNull() && !slot_desc->is_nullable()) { + } else if (is_null && !slot_desc->is_nullable()) { std::string details = absl::Substitute(INVALID_NULL_VALUE, col_name); return Status::RuntimeError(details); } @@ -846,7 +1104,7 @@ Status ScrollParser::fill_columns(const std::vector& slots, const std::string& val = str_col.GetString(); size_t val_size = str_col.GetStringLength(); StringParser::ParseResult result; - bool b = StringParser::string_to_bool(val.c_str(), val_size, &result); + bool b = parse_es_bool(val.c_str(), val_size, &result); RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type); col_ptr->insert_data(const_cast(reinterpret_cast(&b)), 0); break; diff --git a/be/src/exec/es/es_scroll_query.cpp b/be/src/exec/es/es_scroll_query.cpp index fc89341239f993..e99d67a6afdbdd 100644 --- a/be/src/exec/es/es_scroll_query.cpp +++ b/be/src/exec/es/es_scroll_query.cpp @@ -115,6 +115,11 @@ std::string ESScrollQueryBuilder::build(const std::map if (pure_docvalue) { es_query_dsl.AddMember("stored_fields", "_none_", allocator); es_query_dsl.AddMember("docvalue_fields", source_node, allocator); + } else if (auto it = properties.find("read_full_source"); + it != properties.end() && it->second == "true") { + // Source projections fail when an alias includes a source-disabled backing. Requesting + // full source retains enabled backings; disabled hits still have no source to decode. + es_query_dsl.AddMember("_source", true, allocator); } else { es_query_dsl.AddMember("_source", source_node, allocator); } diff --git a/be/test/exec/es_scroll_parser_test.cpp b/be/test/exec/es_scroll_parser_test.cpp index 4f4f53618a0f41..d0b44c038d1d47 100644 --- a/be/test/exec/es_scroll_parser_test.cpp +++ b/be/test/exec/es_scroll_parser_test.cpp @@ -15,20 +15,501 @@ // specific language governing permissions and limitations // under the License. +#include "exec/es/es_scroll_parser.h" + #include #include #include +#include #include +#include "core/assert_cast.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "exec/es/es_scroll_query.h" +#include "runtime/descriptors.h" +#include "util/jsonb_utils.h" + namespace doris { class EsScrollParserTest : public testing::Test { public: void SetUp() override {} void TearDown() override {} + + static TSlotDescriptor value_slot(TPrimitiveType::type type, bool array = false) { + TSlotDescriptor slot; + slot.colName = "value"; + slot.nullIndicatorBit = -1; + if (array) { + TTypeNode node; + node.__set_type(TTypeNodeType::ARRAY); + node.__set_contains_null(true); + node.__set_contains_nulls({true}); + slot.slotType.types.push_back(node); + } + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar; + scalar.__set_type(type); + node.__set_scalar_type(scalar); + slot.slotType.types.push_back(node); + return slot; + } + + static Status read_scalar(TPrimitiveType::type type, const std::string& json, + MutableColumnPtr* column, bool array = false, + const cctz::time_zone& time_zone = cctz::utc_time_zone()) { + auto descriptor = value_slot(type, array); + descriptor.nullIndicatorBit = 0; + SlotDescriptor slot(descriptor); + ScrollParser parser(false); + RETURN_IF_ERROR( + parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + json + "}}]}}", true)); + std::vector slots {&slot}; + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + RETURN_IF_ERROR(parser.fill_columns(slots, columns, &eof, {}, time_zone)); + *column = std::move(columns[0]); + return Status::OK(); + } + + static TSlotDescriptor boolean_slot(bool array) { + return value_slot(TPrimitiveType::BOOLEAN, array); + } }; +TEST_F(EsScrollParserTest, DateSourceEpochMillisStrings) { + for (auto type : {TPrimitiveType::DATE, TPrimitiveType::DATETIME, TPrimitiveType::DATEV2, + TPrimitiveType::DATETIMEV2}) { + SlotDescriptor slot(value_slot(type)); + for (const auto& [input, expected] : std::vector> { + {"0", "1970-01-01 00:00:00"}, + {"0000000000000", "1970-01-01 00:00:00"}, + {"1.5", "1970-01-01 00:00:00"}, + {"1000.", "1970-01-01 00:00:01"}, + {"12345", "1970-01-01 00:00:12"}, + {"-0.000001", "1969-12-31 23:59:59"}, + {"-1000.5", "1969-12-31 23:59:58"}, + {"1677895728000", "2023-03-04 02:08:48"}, + {"2024", "2024-01-01 00:00:00"}, + {"2024-01-01", "2024-01-01 00:00:00"}}) { + SCOPED_TRACE(input); + MutableColumnPtr column; + auto status = read_scalar(type, "\"" + input + "\"", &column); + ASSERT_TRUE(status.ok()) << type << ": " << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + bool date_only = type == TPrimitiveType::DATE || type == TPrimitiveType::DATEV2; + EXPECT_EQ(slot.type()->to_string( + assert_cast(*column).get_nested_column(), 0), + date_only ? expected.substr(0, 10) : expected); + } + } +} + +TEST_F(EsScrollParserTest, DateSourceEpochMillisNumbers) { + SlotDescriptor slot(value_slot(TPrimitiveType::DATETIMEV2)); + for (const auto& [input, expected] : std::vector> { + {"0", "1970-01-01 00:00:00"}, + {"1.5", "1970-01-01 00:00:00"}, + {"1000.9", "1970-01-01 00:00:01"}, + {"-1", "1969-12-31 23:59:59"}, + {"-1000.5", "1969-12-31 23:59:58"}, + {"1677895728000.5", "2023-03-04 02:08:48"}}) { + SCOPED_TRACE(input); + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::DATETIMEV2, input, &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(slot.type()->to_string( + assert_cast(*column).get_nested_column(), 0), + expected); + } +} + +TEST_F(EsScrollParserTest, DateSourceEpochMillisArraysAndTimezone) { + cctz::time_zone time_zone; + ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &time_zone)); + SlotDescriptor slot(value_slot(TPrimitiveType::DATETIMEV2)); + for (const std::string input : {R"(["0"])", "[1.5]"}) { + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::DATETIMEV2, input, &column, false, time_zone); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(slot.type()->to_string( + assert_cast(*column).get_nested_column(), 0), + "1970-01-01 08:00:00"); + } + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::DATETIMEV2, R"([null,"0",-1000.5])", &column, true); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& array = assert_cast( + assert_cast(*column).get_nested_column()); + ASSERT_EQ(array.get_offsets()[0], 3); + EXPECT_TRUE(array.get_data().is_null_at(0)); + const auto& dates = assert_cast(array.get_data()).get_nested_column(); + EXPECT_EQ(slot.type()->to_string(dates, 1), "1970-01-01 00:00:00"); + EXPECT_EQ(slot.type()->to_string(dates, 2), "1969-12-31 23:59:58"); +} + +TEST_F(EsScrollParserTest, DateSourceRejectsInvalidEpochMillis) { + for (const std::string input : {R"("1.2.3")", R"("9223372036854775808")", "1e100", + "18446744073709551615", "253402300800000", "-62167219200001"}) { + MutableColumnPtr column; + EXPECT_FALSE(read_scalar(TPrimitiveType::DATETIMEV2, input, &column).ok()) << input; + } +} + +TEST_F(EsScrollParserTest, JsonbSourcePreservesArrayContainers) { + for (const std::string value : {R"([{"x":1}])", "[]", "[null]", R"({"x":1})"}) { + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::JSONB, value, &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(column->size(), 1); + ASSERT_FALSE(column->is_null_at(0)) << value; + const auto bytes = column->get_data_at(0); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(bytes.data, bytes.size), value); + } +} + +TEST_F(EsScrollParserTest, IntegerNumericDoublePreservesLongBoundary) { + for (const std::string value : {"-9223372036854775808.0", "-9.223372036854776E18"}) { + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::BIGINT, value, &column); + ASSERT_TRUE(status.ok()) << value << ": " << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + EXPECT_EQ(assert_cast(*column).get_nested_column().get_int(0), + INT64_MIN); + } +} + +TEST_F(EsScrollParserTest, NumericSourceEmptyStringsAreNull) { + for (auto type : {TPrimitiveType::TINYINT, TPrimitiveType::SMALLINT, TPrimitiveType::INT, + TPrimitiveType::BIGINT, TPrimitiveType::FLOAT, TPrimitiveType::DOUBLE}) { + for (const std::string value : {R"("")", R"([""])"}) { + MutableColumnPtr column; + auto status = read_scalar(type, value, &column); + ASSERT_TRUE(status.ok()) << type << ": " << value << ": " << status.to_string(); + ASSERT_EQ(column->size(), 1); + EXPECT_TRUE(column->is_null_at(0)); + } + } +} + +TEST_F(EsScrollParserTest, DeclaredSourceArraysPreserveNullElements) { + for (auto type : {TPrimitiveType::BIGINT, TPrimitiveType::BOOLEAN, TPrimitiveType::DOUBLE, + TPrimitiveType::STRING, TPrimitiveType::DATEV2, TPrimitiveType::JSONB}) { + std::string value; + switch (type) { + case TPrimitiveType::BOOLEAN: + value = "true"; + break; + case TPrimitiveType::STRING: + value = R"("text")"; + break; + case TPrimitiveType::DATEV2: + value = R"("2026-01-01")"; + break; + case TPrimitiveType::JSONB: + value = R"({"x":1})"; + break; + default: + value = "1"; + } + MutableColumnPtr column; + auto status = read_scalar(type, "[null," + value + ",null]", &column, true); + ASSERT_TRUE(status.ok()) << type << ": " << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + const auto& array = assert_cast( + assert_cast(*column).get_nested_column()); + ASSERT_EQ(array.get_offsets()[0], 3); + EXPECT_TRUE(array.get_data().is_null_at(0)); + EXPECT_FALSE(array.get_data().is_null_at(1)); + EXPECT_TRUE(array.get_data().is_null_at(2)); + } +} + +TEST_F(EsScrollParserTest, NarrowIntegerSourceAcceptsJavaFloatingGrammar) { + for (auto type : {TPrimitiveType::TINYINT, TPrimitiveType::SMALLINT, TPrimitiveType::INT}) { + for (const std::string value : {"0x1.8p1", "0X1.8P1f", "3d", "+3F", " 3d "}) { + MutableColumnPtr column; + auto status = read_scalar(type, "\"" + value + "\"", &column); + ASSERT_TRUE(status.ok()) << type << ": " << value << ": " << status.to_string(); + EXPECT_EQ(assert_cast(*column).get_nested_column().get_int(0), + 3); + } + for (const std::string value : {"0x3", "0x1.8", "0x1p1junk", "3dd", "3 f"}) { + MutableColumnPtr column; + EXPECT_FALSE(read_scalar(type, "\"" + value + "\"", &column).ok()) << value; + } + } +} + +TEST_F(EsScrollParserTest, FloatingSourceAcceptsJavaFloatingGrammar) { + for (auto type : {TPrimitiveType::FLOAT, TPrimitiveType::DOUBLE}) { + for (const std::string value : {"0x1.8p1", "3d", "+3F"}) { + MutableColumnPtr column; + auto status = read_scalar(type, "\"" + value + "\"", &column); + ASSERT_TRUE(status.ok()) << value << ": " << status.to_string(); + auto bytes = column->get_data_at(0); + if (type == TPrimitiveType::FLOAT) { + float actual; + memcpy(&actual, bytes.data, sizeof(actual)); + EXPECT_FLOAT_EQ(actual, 3.0F); + } else { + double actual; + memcpy(&actual, bytes.data, sizeof(actual)); + EXPECT_DOUBLE_EQ(actual, 3.0); + } + } + } +} + +TEST_F(EsScrollParserTest, FloatSourceRoundsDirectlyToFloat) { + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::FLOAT, + R"("1.000000059604644775390625000000000000000000000001")", &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto bytes = column->get_data_at(0); + float actual; + memcpy(&actual, bytes.data, sizeof(actual)); + EXPECT_EQ(actual, 1.00000011920928955078125F); +} + +TEST_F(EsScrollParserTest, EmptyAndNullScalarSourceArrays) { + for (auto type : {TPrimitiveType::FLOAT, TPrimitiveType::DOUBLE, TPrimitiveType::BOOLEAN, + TPrimitiveType::BIGINT}) { + for (const std::string value : {"[]", "[null]", "null"}) { + MutableColumnPtr column; + auto status = read_scalar(type, value, &column); + ASSERT_TRUE(status.ok()) << type << ": " << value << ": " << status.to_string(); + ASSERT_EQ(column->size(), 1); + EXPECT_TRUE(column->is_null_at(0)); + } + } +} + +TEST_F(EsScrollParserTest, FloatingPointSourceSingletonArrays) { + for (auto type : {TPrimitiveType::FLOAT, TPrimitiveType::DOUBLE}) { + MutableColumnPtr column; + auto status = read_scalar(type, "[1.5]", &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + auto data = assert_cast(*column).get_nested_column().get_data_at(0); + if (type == TPrimitiveType::FLOAT) { + float actual; + memcpy(&actual, data.data, sizeof(actual)); + EXPECT_FLOAT_EQ(actual, 1.5F); + } else { + double actual; + memcpy(&actual, data.data, sizeof(actual)); + EXPECT_DOUBLE_EQ(actual, 1.5); + } + } +} + +TEST_F(EsScrollParserTest, BooleanAndIntegerSourceSingletonArrays) { + MutableColumnPtr column; + auto status = read_scalar(TPrimitiveType::BOOLEAN, R"([""])", &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + EXPECT_FALSE(column->get_bool(0)); + status = read_scalar(TPrimitiveType::BIGINT, "[1]", &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(column->is_null_at(0)); + EXPECT_EQ(assert_cast(*column).get_nested_column().get_int(0), 1); +} + +TEST_F(EsScrollParserTest, NarrowIntegerNaNAndInfinity) { + for (auto type : {TPrimitiveType::TINYINT, TPrimitiveType::SMALLINT, TPrimitiveType::INT}) { + MutableColumnPtr column; + auto status = read_scalar(type, R"("NaN")", &column); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(assert_cast(*column).get_nested_column().get_int(0), 0); + EXPECT_FALSE(read_scalar(type, R"("Infinity")", &column).ok()); + } +} + +TEST_F(EsScrollParserTest, NarrowIntegerStringRounding) { + for (auto type : {TPrimitiveType::TINYINT, TPrimitiveType::SMALLINT, TPrimitiveType::INT, + TPrimitiveType::BIGINT}) { + SlotDescriptor slot(value_slot(type)); + std::vector slots {&slot}; + for (const auto& [value, expected] : std::vector> { + {"0.99999999999999999", type == TPrimitiveType::BIGINT ? 0 : 1}, + {"-0.99999999999999999", type == TPrimitiveType::BIGINT ? 0 : -1}, + {"1.2e1", 12}, + {"127.9", 127}}) { + ScrollParser parser(false); + ASSERT_TRUE( + parser.parse(R"({"hits":{"hits":[{"_source":{"value":")" + value + R"("}}]}})", + true) + .ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + auto status = parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(columns[0]->get_int(0), expected) << type << ": " << value; + } + } +} + +TEST_F(EsScrollParserTest, IntegerSourceCoercion) { + SlotDescriptor slot(value_slot(TPrimitiveType::BIGINT)); + std::vector slots {&slot}; + for (const auto& [value, expected] : + std::vector> {{R"("1.2e3")", 1200}, + {R"("-1.9E1")", -19}, + {R"("1.9e-1")", 0}, + {R"(".9e1")", 9}, + {R"("00012.9")", 12}, + {R"("-0.9")", 0}, + {R"("9223372036854775807")", INT64_MAX}, + {R"("-9.223372036854775808e18")", INT64_MIN}, + {R"("922337203685477580.7e1")", INT64_MAX}, + {"1.9", 1}, + {"[5]", 5}, + {R"(["1.2e3"])", 1200}}) { + ScrollParser parser(false); + ASSERT_TRUE(parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + value + "}}]}}", true) + .ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + auto status = parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << value << ": " << status.to_string(); + ASSERT_EQ(columns[0]->size(), 1); + EXPECT_EQ(columns[0]->get_int(0), expected) << value; + } + for (const std::string value : + {R"("9223372036854775808")", R"("1e1000000")", R"("1e")", R"("1.2junk")", "[1,2]"}) { + ScrollParser parser(false); + ASSERT_TRUE(parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + value + "}}]}}", true) + .ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + EXPECT_FALSE(parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()).ok()) + << value; + } +} + +TEST_F(EsScrollParserTest, FullSourceRequestOnlyAffectsSourceMode) { + std::map properties { + {"query_dsl", "{\"match_all\":{}}"}, {"batch_size", "100"}, {"doc_values_mode", "0"}}; + bool docvalues = true; + rapidjson::Document body; + auto query = ESScrollQueryBuilder::build(properties, {"value"}, {}, &docvalues); + body.Parse(query.c_str()); + ASSERT_FALSE(body.HasParseError()); + EXPECT_TRUE(body["_source"].IsArray()); + EXPECT_FALSE(docvalues); + properties["read_full_source"] = "true"; + query = ESScrollQueryBuilder::build(properties, {"value"}, {}, &docvalues); + body.Parse(query.c_str()); + ASSERT_TRUE(body["_source"].IsBool()); + EXPECT_TRUE(body["_source"].GetBool()); + properties["doc_values_mode"] = "1"; + query = ESScrollQueryBuilder::build(properties, {"value"}, {{"value", "value.raw"}}, + &docvalues); + body.Parse(query.c_str()); + EXPECT_TRUE(docvalues); + EXPECT_FALSE(body.HasMember("_source")); + ASSERT_TRUE(body["docvalue_fields"].IsArray()); + EXPECT_STREQ(body["docvalue_fields"][0].GetString(), "value.raw"); +} + +TEST_F(EsScrollParserTest, IntegerArraySourceCoercion) { + SlotDescriptor slot(value_slot(TPrimitiveType::BIGINT, true)); + std::vector slots {&slot}; + ScrollParser parser(false); + ASSERT_TRUE( + parser.parse(R"({"hits":{"hits":[{"_source":{"value":["1.2e3",-1.9,"9223372036854775807"]}}]}})", + true) + .ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + auto status = parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& array = assert_cast(*columns[0]); + const auto& values = assert_cast(array.get_data()).get_nested_column(); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(values.get_int(0), 1200); + EXPECT_EQ(values.get_int(1), -1); + EXPECT_EQ(values.get_int(2), INT64_MAX); +} + +TEST_F(EsScrollParserTest, IntegerCoercionChecksNarrowBounds) { + SlotDescriptor slot(value_slot(TPrimitiveType::TINYINT)); + std::vector slots {&slot}; + for (const std::string value : {"128", "-129", R"("1.28e2")", R"("-1.29e2")"}) { + ScrollParser parser(false); + ASSERT_TRUE(parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + value + "}}]}}", true) + .ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + EXPECT_FALSE(parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()).ok()) + << value; + } +} + +TEST_F(EsScrollParserTest, BooleanSourceRepresentations) { + SlotDescriptor slot(boolean_slot(false)); + std::vector slots {&slot}; + for (const auto& [value, expected] : + std::vector> {{R"("")", false}, + {R"("false")", false}, + {R"("true")", true}, + {"false", false}, + {"true", true}}) { + ScrollParser parser(false); + auto status = + parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + value + "}}]}}", true); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + status = parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(columns[0]->size(), 1); + EXPECT_EQ(columns[0]->get_bool(0), expected); + } +} + +TEST_F(EsScrollParserTest, BooleanArraySourceEmptyString) { + SlotDescriptor slot(boolean_slot(true)); + std::vector slots {&slot}; + ScrollParser parser(false); + auto status = + parser.parse(R"({"hits":{"hits":[{"_source":{"value":["","true",false]}}]}})", true); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + status = parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& array = assert_cast(*columns[0]); + const auto& values = assert_cast(array.get_data()).get_nested_column(); + ASSERT_EQ(values.size(), 3); + EXPECT_FALSE(values.get_bool(0)); + EXPECT_TRUE(values.get_bool(1)); + EXPECT_FALSE(values.get_bool(2)); +} + +TEST_F(EsScrollParserTest, InvalidBooleanSourceIsRejected) { + SlotDescriptor slot(boolean_slot(false)); + std::vector slots {&slot}; + ScrollParser parser(false); + ASSERT_TRUE(parser.parse(R"({"hits":{"hits":[{"_source":{"value":"invalid"}}]}})", true).ok()); + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + EXPECT_FALSE(parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()).ok()); +} + // Test timezone pattern matching for ES datetime parsing fix TEST_F(EsScrollParserTest, TestTimezonePatternMatching) { RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index b3fad979a2e452..b9f15734eef8e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.es.EsFieldContext; import org.apache.doris.datasource.es.EsMetaStateTracker; import org.apache.doris.datasource.es.EsRestClient; import org.apache.doris.datasource.es.EsTablePartitions; @@ -164,6 +165,11 @@ public List needCompatDateFields() throws UserException { return esMetaStateTracker.searchContext().needCompatDateFields(); } + public EsFieldContext fieldContext() throws UserException { + initEsMetaStateTracker(); + return esMetaStateTracker.searchContext().fieldContext(); + } + private void initEsMetaStateTracker() { if (esMetaStateTracker == null) { esMetaStateTracker = new EsMetaStateTracker(client, this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java new file mode 100644 index 00000000000000..04601f47f34aa7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Holds field-level context derived from ES index mapping analysis. + * + *
    + *
  • {@code fetchFieldsContext} — maps text fields to their keyword sub-fields + * (e.g. "city" → "city.raw"), used when {@code enable_keyword_sniff = true}.
  • + *
  • {@code docValueFieldsContext} — maps columns to doc_value-capable field paths, + * used when {@code enable_docvalue_scan = true}.
  • + *
  • {@code needCompatDateFields} — columns using default/strict_date_optional_time format + * that need special date compatibility handling.
  • + *
+ */ +public class EsFieldContext { + + private final Map fetchFieldsContext = new HashMap<>(); + private final Map docValueFieldsContext = new HashMap<>(); + private final List needCompatDateFields = new ArrayList<>(); + private final Set sourceValueUnsafeFields = new HashSet<>(); + private final Set sourceModeUnsafeFields = new HashSet<>(); + private boolean requiresFullSourceRead; + + private final Map column2typeMap = new HashMap<>(); + + public Map getFetchFieldsContext() { + return fetchFieldsContext; + } + + public Map getDocValueFieldsContext() { + return docValueFieldsContext; + } + + public List getNeedCompatDateFields() { + return needCompatDateFields; + } + + /** + * Returns column name → ES raw type string mapping (e.g., "keyword", "text", "long"). + * Used by query DSL builder to determine whether LIKE pushdown is safe on keyword fields. + */ + public Map getColumn2typeMap() { + return column2typeMap; + } + + public Set getSourceValueUnsafeFields() { + return sourceValueUnsafeFields; + } + + public Set getSourceModeUnsafeFields() { + return sourceModeUnsafeFields; + } + + public boolean requiresFullSourceRead() { + return requiresFullSourceRead; + } + + public void requireFullSourceRead() { + requiresFullSourceRead = true; + } + + @Override + public String toString() { + return "EsFieldContext{" + + "fetchFields=" + fetchFieldsContext.size() + + ", docValueFields=" + docValueFieldsContext.size() + + ", compatDateFields=" + needCompatDateFields.size() + + ", column2type=" + column2typeMap.size() + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java new file mode 100644 index 00000000000000..f2f54c166fde7a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java @@ -0,0 +1,441 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * ES mapping parsing and field resolution utilities. + */ +public final class EsMappingUtils { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static final Set DEFAULT_DOCVALUE_DISABLED_FIELDS = + new HashSet<>(Arrays.asList("text", "annotated_text", "match_only_text")); + + private static final Set ROOT_MAPPING_OBJECT_PARAMETERS = new HashSet<>(Arrays.asList( + "properties", "runtime", "_source", "_routing", "_field_names", "_size", "_data_stream_timestamp")); + + private static final Set DOCVALUE_QUERY_TYPES = new HashSet<>(Arrays.asList( + "keyword", "byte", "short", "integer", "long", "unsigned_long", "float", "double", + "half_float", "scaled_float", "date", "date_nanos", "boolean", "ip", "geo_point")); + + private static final Set IGNORE_MALFORMED_TYPES = new HashSet<>(Arrays.asList( + "byte", "short", "integer", "long", "unsigned_long", "float", "double", "half_float", "scaled_float", + "boolean", "date", "date_nanos", "ip", "geo_point", "geo_shape")); + + private EsMappingUtils() { + } + + /** + * Parse the top-level "mappings" node from an index mapping JSON response. + */ + public static ObjectNode getMapping(String indexMapping) { + try { + ObjectNode jsonNodes = (ObjectNode) MAPPER.readTree(indexMapping); + // Alias mapping response order is unspecified. A bare/typed empty backing can appear + // first; select a populated schema without discarding its _meta array annotations. + for (JsonNode index : jsonNodes) { + ObjectNode mapping = (ObjectNode) index.get("mappings"); + ObjectNode root = getRootSchema(mapping.deepCopy(), null, new ArrayList<>(), true); + if (root.path("properties").size() > 0) { + return mapping; + } + } + // Preserve the existing diagnostic when no backing index provides a usable schema. + return (ObjectNode) jsonNodes.iterator().next().get("mappings"); + } catch (IOException e) { + throw new DorisEsException("Failed to parse ES mapping: " + e.getMessage()); + } + } + + /** + * Extract the root schema from mapping node, handling ES 5.x/6.x/7.x/8.x differences. + */ + public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType, + List arrayFields) { + return getRootSchema(mappings, mappingType, arrayFields, false); + } + + private static ObjectNode getRootSchema(ObjectNode mappings, String mappingType, + List arrayFields, boolean allowEmpty) { + if (mappingType == null) { + checkNonPropertiesFields(mappings, arrayFields, allowEmpty); + // Alias safety analysis must accept backing indices that have not acquired a mapping yet. + if (mappings.isEmpty()) { + return mappings; + } + String firstType = mappings.fieldNames().next(); + // A typeless mapping may contain only flags or root objects such as _source. + // Type wrappers are objects; legacy type names may also be names of scalar settings. + Iterator keys = mappings.fieldNames(); + while (keys.hasNext()) { + String key = keys.next(); + JsonNode value = mappings.get(key); + // ES6 permits a type named runtime. Its properties contain field descriptors, + // whereas a runtime field named properties has scalar parameters such as type. + boolean legacyRuntime = "runtime".equals(key) && value.path("properties").isObject() + && allValuesAreObjects(value.path("properties")); + if (!value.isObject() || (ROOT_MAPPING_OBJECT_PARAMETERS.contains(key) && !legacyRuntime)) { + return mappings; + } + } + ObjectNode firstData = (ObjectNode) mappings.get(firstType); + checkNonPropertiesFields(firstData, arrayFields, allowEmpty); + return firstData; + } else { + if (mappings.has(mappingType)) { + ObjectNode jsonData = (ObjectNode) mappings.get(mappingType); + checkNonPropertiesFields(jsonData, arrayFields, allowEmpty); + return jsonData; + } + return getRootSchema(mappings, null, arrayFields, allowEmpty); + } + } + + private static boolean allValuesAreObjects(JsonNode node) { + for (JsonNode value : node) { + if (!value.isObject()) { + return false; + } + } + return true; + } + + /** + * Get mapping properties ObjectNode for a given index. + */ + public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, + String mappingType) { + ObjectNode mappings = getMapping(indexMapping); + ObjectNode rootSchema = getRootSchema(mappings, mappingType, new ArrayList<>()); + ObjectNode properties = (ObjectNode) rootSchema.get("properties"); + if (properties == null) { + throw new DorisEsException( + "index[" + sourceIndex + "] type[" + mappingType + + "] mapping not found for the ES Cluster"); + } + return properties; + } + + /** + * Resolve field contexts (keyword sniff, doc_value, date compat) from mapping properties. + * + * @param columnNames column names to resolve + * @param sourceIndex index name for error messages + * @param indexMapping raw mapping JSON + * @param mappingType mapping type (null for ES 7+) + * @return populated EsFieldContext + */ + public static EsFieldContext resolveFieldContext(List columnNames, + String sourceIndex, String indexMapping, String mappingType) { + EsFieldContext fieldContext = new EsFieldContext(); + ObjectNode properties = getMappingProps(sourceIndex, indexMapping, mappingType); + + // When columnNames is empty, resolve all fields from the mapping. + // This ensures column2typeMap, keyword sniff, docvalue, and date compat contexts + // are populated even when specific column handles are not provided. + Iterable fieldsToResolve; + if (columnNames == null || columnNames.isEmpty()) { + List allFields = new ArrayList<>(); + Iterator iter = properties.fieldNames(); + while (iter.hasNext()) { + allFields.add(iter.next()); + } + fieldsToResolve = allFields; + } else { + fieldsToResolve = columnNames; + } + + for (String colName : fieldsToResolve) { + if ("_id".equals(colName)) { + continue; + } + if (!properties.has(colName)) { + throw new DorisEsException( + "index[" + sourceIndex + "] mapping[" + indexMapping + + "] not found column " + colName + " for the ES Cluster"); + } + ObjectNode fieldObject = (ObjectNode) properties.get(colName); + if (!fieldObject.has("type")) { + fieldContext.getColumn2typeMap().put(colName, "object"); + continue; + } + String fieldType = fieldObject.get("type").asText(); + fieldContext.getColumn2typeMap().put(colName, fieldType); + resolveDateFields(fieldContext, fieldObject, colName, fieldType); + resolveDocValuesFields(fieldContext, fieldObject, colName, fieldType); + } + resolveSourceValueSafety(fieldContext, indexMapping, mappingType, properties); + return fieldContext; + } + + private static void resolveSourceValueSafety(EsFieldContext context, String indexMapping, String mappingType, + ObjectNode schemaProperties) { + try { + // Mapping-induced source/index differences on any alias backing index must prevent + // filtering or replacing source values through that path. + JsonNode indices = MAPPER.readTree(indexMapping); + Set columns = new HashSet<>(context.getColumn2typeMap().keySet()); + // Use the same populated schema as inference, including unprojected filter columns. + schemaProperties.fieldNames().forEachRemaining(columns::add); + // Filters can reference columns absent from the projection. Resolve their query paths + // from the exposed schema too; the projection still controls docvalue read eligibility. + for (String column : columns) { + ObjectNode field = (ObjectNode) schemaProperties.get(column); + resolveKeywordFields(context, field, column, field.path("type").asText()); + } + Set unsafeDocValues = new HashSet<>(); + for (JsonNode index : indices) { + ObjectNode root = getRootSchema((ObjectNode) index.get("mappings"), mappingType, + new ArrayList<>(), true); + JsonNode mappingDefaults = index.path("settings").path("index").path("mapping"); + JsonNode properties = root.path("properties"); + if (isSourceDisabled(root.path("_source"))) { + // A source projection fails on disabled backings; full source preserves the + // enabled backings of a mixed alias while disabled hits remain NULL. + context.requireFullSourceRead(); + } + Set copyTargets = new HashSet<>(); + collectCopyTargets(properties, copyTargets); + for (String column : columns) { + JsonNode field = properties.path(column); + JsonNode queryField = fieldAtPath(field, column, context.getFetchFieldsContext().get(column)); + JsonNode readField = fieldAtPath(field, column, context.getDocValueFieldsContext().get(column)); + // Disabled root parsing leaves source fields but produces no index/docvalues, + // even if an explicit properties section still describes those fields. + boolean sourceChanged = !root.path("enabled").asBoolean(true) + || filtersSourceColumn(root.path("_source"), column) + || root.path("runtime").has(column) || copyTargets.contains(column); + // ES mappings do not distinguish arrays from scalars. Even a short equality can + // match one indexed array element while Doris reads the entire source JSON string. + if (sourceChanged || changesSourceValue(field, mappingDefaults) + || changesSourceValue(queryField, mappingDefaults) + || "text".equals(queryField.path("type").asText()) || !isSearchable(queryField)) { + context.getSourceValueUnsafeFields().add(column); + } + // Source can omit generated/aliased values, retain unrounded numbers or decode + // default dates in local time. Docvalues remain eligible for these mappings. + if (requiresSourcePredicateEvaluation(field) || requiresSourcePredicateEvaluation(queryField)) { + context.getSourceModeUnsafeFields().add(column); + } + // Query and docvalue paths may select different multi-fields. An unused sibling's + // normalizer must not invalidate either path; only the chosen read value matters. + if (sourceChanged || changesSourceValue(field, mappingDefaults) + || changesSourceValue(readField, mappingDefaults) + || !readField.path("doc_values").asBoolean(true) + || DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) { + // One backing without this docvalue path would return missing values for its hits. + unsafeDocValues.add(column); + } + } + } + unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove); + } catch (IOException e) { + throw new DorisEsException("Failed to parse ES mapping: " + e.getMessage()); + } + } + + private static JsonNode fieldAtPath(JsonNode field, String column, String path) { + return path == null || column.equals(path) ? field + : field.path("fields").path(path.substring(column.length() + 1)); + } + + private static boolean changesSourceValue(JsonNode field, JsonNode mappingDefaults) { + // With index-level ignore_malformed=true/coerce=false, source "5" survives without an + // indexed value. A field's explicit false overrides that default; keyword ignores it. + boolean ignoresMalformed = IGNORE_MALFORMED_TYPES.contains(field.path("type").asText()) + && field.path("ignore_malformed").asBoolean(mappingDefaults.path("ignore_malformed").asBoolean(false)); + return field.isMissingNode() || field.has("ignore_above") || field.hasNonNull("normalizer") + || field.hasNonNull("null_value") || ignoresMalformed; + } + + private static boolean requiresSourcePredicateEvaluation(JsonNode field) { + String type = field.path("type").asText(); + // A default date may be a date-only source string read as local midnight, unlike its UTC index value. + return "scaled_float".equals(type) || "half_float".equals(type) || "alias".equals(type) + || field.hasNonNull("script") || ("date".equals(type) && !field.has("format")); + } + + private static boolean isSearchable(JsonNode field) { + // Numeric, date, boolean, IP and keyword docvalues remain queryable without an inverted + // index. Searchability is independent of the source/read-value equivalence checks above. + return field.path("index").asBoolean(true) + || (DOCVALUE_QUERY_TYPES.contains(field.path("type").asText()) + && field.path("doc_values").asBoolean(true)); + } + + private static boolean filtersSourceColumn(JsonNode source, String column) { + JsonNode includes = source.path("includes"); + return isSourceDisabled(source) + || (includes.size() > 0 && !matchesSourcePattern(includes, column)) + || matchesSourcePattern(source.path("excludes"), column); + } + + private static boolean isSourceDisabled(JsonNode source) { + // ES can serialize disabled source as mode=disabled without an enabled flag. + return !source.path("enabled").asBoolean(true) || "disabled".equals(source.path("mode").asText()); + } + + private static boolean matchesSourcePattern(JsonNode patterns, String column) { + for (JsonNode pattern : patterns) { + // Source filters use '*' globs, not SQL LIKE or regular expressions. Treat every + // other character literally and avoid regex backtracking on mapping-supplied patterns. + String[] parts = pattern.asText().split("\\*", -1); + if (parts.length == 1) { + if (column.equals(parts[0])) { + return true; + } + continue; + } + if (!column.startsWith(parts[0])) { + continue; + } + int offset = parts[0].length(); + for (int i = 1; i < parts.length - 1 && offset >= 0; i++) { + int found = column.indexOf(parts[i], offset); + offset = found < 0 ? -1 : found + parts[i].length(); + } + String suffix = parts[parts.length - 1]; + if (offset >= 0 && offset + suffix.length() <= column.length() && column.endsWith(suffix)) { + return true; + } + } + return false; + } + + private static void collectCopyTargets(JsonNode properties, Set targets) { + for (JsonNode field : properties) { + JsonNode copyTo = field.path("copy_to"); + if (copyTo.isTextual()) { + targets.add(copyTo.asText()); + } else if (copyTo.isArray()) { + copyTo.forEach(target -> targets.add(target.asText())); + } + collectCopyTargets(field.path("properties"), targets); + collectCopyTargets(field.path("fields"), targets); + } + } + + private static void resolveDateFields(EsFieldContext context, ObjectNode fieldObject, + String colName, String fieldType) { + if ("date".equals(fieldType)) { + if (!fieldObject.has("format") + || "strict_date_optional_time".equals(fieldObject.get("format").asText())) { + context.getNeedCompatDateFields().add(colName); + } + } + } + + private static void resolveKeywordFields(EsFieldContext context, ObjectNode fieldObject, + String colName, String fieldType) { + if ("text".equals(fieldType)) { + JsonNode fieldsObject = fieldObject.get("fields"); + if (fieldsObject != null) { + Iterator fieldNames = fieldsObject.fieldNames(); + while (fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); + if ("keyword".equals(innerTypeObject.get("type").asText())) { + context.getFetchFieldsContext() + .put(colName, colName + "." + fieldName); + } + } + } + } + } + + private static void resolveDocValuesFields(EsFieldContext context, ObjectNode fieldObject, + String colName, String fieldType) { + String docValueField = null; + if (DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { + JsonNode fieldsObject = fieldObject.get("fields"); + if (fieldsObject != null) { + Iterator fieldNames = fieldsObject.fieldNames(); + while (fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); + if (DEFAULT_DOCVALUE_DISABLED_FIELDS + .contains(innerTypeObject.get("type").asText())) { + continue; + } + if (innerTypeObject.has("doc_values")) { + boolean docValue = innerTypeObject.get("doc_values").asBoolean(); + if (docValue) { + // A multi-field has its own path; the parent text has no docvalues. + docValueField = colName + "." + fieldName; + } + } else if (innerTypeObject.has("ignore_above")) { + // keyword with ignore_above — cannot rely on doc_values + } else { + docValueField = colName + "." + fieldName; + } + } + } + } else { + if (fieldObject.has("doc_values")) { + boolean docValue = fieldObject.get("doc_values").asBoolean(); + if (!docValue) { + return; + } + } else if (fieldType == null || "nested".equals(fieldType)) { + return; + } else if (fieldObject.has("ignore_above")) { + return; + } + docValueField = colName; + } + if (docValueField != null && !docValueField.isEmpty()) { + context.getDocValueFieldsContext().put(colName, docValueField); + } + } + + private static void checkNonPropertiesFields(ObjectNode mappings, + List arrayFields, boolean allowEmpty) { + JsonNode metaNode = mappings.remove("_meta"); + if (metaNode != null) { + JsonNode dorisMeta = metaNode.get("doris"); + if (dorisMeta != null) { + JsonNode arrayNode = dorisMeta.get("array_fields"); + if (arrayNode != null) { + for (JsonNode jsonNode : arrayNode) { + arrayFields.add(jsonNode.asText()); + } + } + } + } + mappings.remove("dynamic_templates"); + mappings.remove("dynamic"); + mappings.remove("_default_"); + if (!allowEmpty && mappings.isEmpty()) { + throw new DorisEsException( + "Do not support index without explicit mapping."); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java index e2c7fa0b688913..d5fdbfa1222e05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java @@ -120,7 +120,8 @@ public Map getHttpNodes() throws DorisEsException { * Get mapping for indexName. */ public String getMapping(String indexName) throws DorisEsException { - String path = indexName + "/_mapping"; + // Index-level defaults affect whether indexed/docvalue values preserve _source semantics. + String path = indexName + "?filter_path=*.mappings,*.settings.index.mapping.*"; String indexMapping = execute(path); if (indexMapping == null) { throw new DorisEsException("index[" + indexName + "] not found"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java index f1e907bfab9095..8682a4f4526df4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java @@ -26,9 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.JsonUtil; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -39,9 +37,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.StreamSupport; /** * Util for ES, some static method. @@ -98,90 +93,16 @@ public static boolean getBoolean(Map properties, String name) th @VisibleForTesting public static ObjectNode getMapping(String indexMapping) { - ObjectNode jsonNodes = JsonUtil.parseObject(indexMapping); - // If the indexName use alias takes the first mapping - return (ObjectNode) jsonNodes.iterator().next().get("mappings"); + return EsMappingUtils.getMapping(indexMapping); } @VisibleForTesting public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType, List arrayFields) { - // Type is null in the following three cases - // 1. Equal 6.8.x and after - // 2. Multi-catalog auto infer - // 3. Equal 6.8.x and before user not passed - if (mappingType == null) { - // remove dynamic templates, for ES 7.x and 8.x - checkNonPropertiesFields(mappings, arrayFields); - String firstType = mappings.fieldNames().next(); - //The first parameter may not be properties, so we need to first determine whether it is 7.x or above. - if (StreamSupport.stream(Spliterators - .spliteratorUnknownSize(mappings.fieldNames(), Spliterator.ORDERED), false) - .anyMatch(s -> s.contains("properties"))) { - // Equal 7.x and after - return mappings; - } else { - ObjectNode firstData = (ObjectNode) mappings.get(firstType); - // check for ES 6.x and before - checkNonPropertiesFields(firstData, arrayFields); - return firstData; - } - } else { - if (mappings.has(mappingType)) { - ObjectNode jsonData = (ObjectNode) mappings.get(mappingType); - // check for ES 6.x and before - checkNonPropertiesFields(jsonData, arrayFields); - return jsonData; - } - // Compatible type error - return getRootSchema(mappings, null, arrayFields); - } - } - - /** - * Check non properties fields - * - * @param mappings - */ - private static void checkNonPropertiesFields(ObjectNode mappings, List arrayFields) { - // remove `_meta` field and parse array_fields - JsonNode metaNode = mappings.remove("_meta"); - if (metaNode != null) { - JsonNode dorisMeta = metaNode.get("doris"); - if (dorisMeta != null) { - JsonNode arrayNode = dorisMeta.get("array_fields"); - if (arrayNode != null) { - for (JsonNode jsonNode : arrayNode) { - arrayFields.add(jsonNode.asText()); - } - } - } - } - // remove `dynamic_templates` field - mappings.remove("dynamic_templates"); - // remove `dynamic` field - mappings.remove("dynamic"); - // remove `_default` field, we do not parse `_default_` mapping, only explicit mapping. - // `_default` _mapping type is deprecated in 7.0 and removed in 8.0 - // https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html - mappings.remove("_default_"); - // check explicit mapping - if (mappings.isEmpty()) { - throw new DorisEsException("Do not support index without explicit mapping."); - } + return EsMappingUtils.getRootSchema(mappings, mappingType, arrayFields); } - /** - * Get mapping properties transform to ObjectNode. - **/ public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, String mappingType) { - ObjectNode mappings = getMapping(indexMapping); - ObjectNode rootSchema = getRootSchema(mappings, mappingType, new ArrayList<>()); - ObjectNode properties = (ObjectNode) rootSchema.get("properties"); - if (properties == null) { - throw new DorisEsException( - "index[" + sourceIndex + "] type[" + mappingType + "] mapping not found for the ES Cluster"); - } - return properties; + return EsMappingUtils.getMappingProps(sourceIndex, indexMapping, mappingType); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java index 9e14f2721269f3..5f9f704389412a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java @@ -18,13 +18,8 @@ package org.apache.doris.datasource.es; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.EsTable; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang3.StringUtils; - -import java.util.Iterator; +import java.util.stream.Collectors; /** * Get index mapping from remote ES Cluster, and resolved `keyword` and `doc_values` field @@ -58,122 +53,9 @@ public void postProcess(SearchContext context) { * @param indexMapping the return value of _mapping */ public static void resolveFields(SearchContext searchContext, String indexMapping) throws DorisEsException { - ObjectNode properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type()); - for (Column col : searchContext.columns()) { - String colName = col.getName(); - // _id not exist mapping, but be can query it. - if (!"_id".equals(colName)) { - if (!properties.has(colName)) { - throw new DorisEsException( - "index[" + searchContext.sourceIndex() + "] mapping[" + indexMapping + "] not found " - + "column " + colName + " for the ES Cluster"); - } - ObjectNode fieldObject = (ObjectNode) properties.get(colName); - if (!fieldObject.has("type")) { - continue; - } - String fieldType = fieldObject.get("type").asText(); - resolveDateFields(searchContext, fieldObject, colName, fieldType); - resolveKeywordFields(searchContext, fieldObject, colName, fieldType); - resolveDocValuesFields(searchContext, fieldObject, colName, fieldType); - } - } - } - - private static void resolveDateFields(SearchContext searchContext, ObjectNode fieldObject, String colName, - String fieldType) { - // Compat use default/strict_date_optional_time format date type, need transform datetime to - if ("date".equals(fieldType)) { - if (!fieldObject.has("format") || "strict_date_optional_time".equals(fieldObject.get("format").asText())) { - searchContext.needCompatDateFields().add(colName); - } - } + EsFieldContext fields = EsMappingUtils.resolveFieldContext(searchContext.columns().stream() + .map(Column::getName).collect(Collectors.toList()), searchContext.sourceIndex(), + indexMapping, searchContext.type()); + searchContext.setFieldContext(fields); } - - - // get a field of keyword type in the fields - private static void resolveKeywordFields(SearchContext searchContext, ObjectNode fieldObject, String colName, - String fieldType) { - // string-type field used keyword type to generate predicate - // if text field type seen, we should use the `field` keyword type? - if ("text".equals(fieldType)) { - JsonNode fieldsObject = fieldObject.get("fields"); - if (fieldsObject != null) { - Iterator fieldNames = fieldsObject.fieldNames(); - while (fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); - // just for text type - if ("keyword".equals(innerTypeObject.get("type").asText())) { - searchContext.fetchFieldsContext().put(colName, colName + "." + fieldName); - } - } - } - } - } - - private static void resolveDocValuesFields(SearchContext searchContext, ObjectNode fieldObject, String colName, - String fieldType) { - String docValueField = null; - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JsonNode fieldsObject = fieldObject.get("fields"); - if (fieldsObject != null) { - Iterator fieldNames = fieldsObject.fieldNames(); - while (fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.get("type").asText())) { - continue; - } - if (innerTypeObject.has("doc_values")) { - boolean docValue = innerTypeObject.get("doc_values").asBoolean(); - if (docValue) { - docValueField = colName; - } - } else if (innerTypeObject.has("ignore_above")) { - // reference: - // https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params - // > ignore_above - // > Do not index any string longer than this value. Defaults to 2147483647 so that all values - // > would be accepted. Please however note that default dynamic mapping rules create a sub - // > keyword field that overrides this default by setting ignore_above: 256. - // this field has `ignore_above` param - // Strings longer than the ignore_above setting will not be indexed or stored - // so we cannot rely on its doc_values - } else { - // a : {c : {}} -> a -> a.c - docValueField = colName + "." + fieldName; - } - } - } - } else { - // set doc_value = false manually - if (fieldObject.has("doc_values")) { - boolean docValue = fieldObject.get("doc_values").asBoolean(); - if (!docValue) { - return; - } - } else if (fieldType == null || "nested".equals(fieldType)) { - // The object field has no type, and nested not support doc value. - return; - } else if (fieldObject.has("ignore_above")) { - // reference: - // https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params - // > ignore_above - // > Do not index any string longer than this value. Defaults to 2147483647 so that all values - // > would be accepted. Please however note that default dynamic mapping rules create a sub - // > keyword field that overrides this default by setting ignore_above: 256. - // this field has `ignore_above` param - // Strings longer than the ignore_above setting will not be indexed or stored - // so we cannot rely on its doc_values - return; - } - docValueField = colName; - } - // docValueField Cannot be null - if (StringUtils.isNotEmpty(docValueField)) { - searchContext.docValueFieldsContext().put(colName, docValueField); - } - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java index 40c86766037f10..5d7f9796605623 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java @@ -57,6 +57,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; @@ -100,12 +101,46 @@ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownLis return null; } case NOT: { - QueryBuilder child = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, + Expr operand = compoundPredicate.getChild(0); + Expr normalized = null; + // Equality can be inverted, but NOT(a < 2) is not a >= 2 for an ES array [1, 2]. + if (operand instanceof CompoundPredicate) { + CompoundPredicate child = (CompoundPredicate) operand; + if (child.getOp() == CompoundPredicate.Operator.NOT) { + normalized = child.getChild(0); + } else { + normalized = new CompoundPredicate(child.getOp() == CompoundPredicate.Operator.AND + ? CompoundPredicate.Operator.OR : CompoundPredicate.Operator.AND, + new CompoundPredicate(CompoundPredicate.Operator.NOT, child.getChild(0), null), + new CompoundPredicate(CompoundPredicate.Operator.NOT, child.getChild(1), null)); + } + } else if (operand instanceof BinaryPredicate) { + BinaryPredicate predicate = (BinaryPredicate) operand; + if (predicate.getOp() == BinaryPredicate.Operator.EQ + || predicate.getOp() == BinaryPredicate.Operator.NE) { + normalized = predicate.negate(); + } + } else if (operand instanceof IsNullPredicate) { + normalized = operand.negate(); + } + if (normalized != null) { + return toEsDsl(normalized, notPushDownList, fieldsContext, builderOptions, column2typeMap); + } + QueryBuilder child = toEsDsl(operand, notPushDownList, fieldsContext, builderOptions, column2typeMap); - if (child != null) { - return QueryBuilders.boolQuery().mustNot(child); + if (child == null) { + return null; } - return null; + BoolQueryBuilder negated = QueryBuilders.boolQuery().mustNot(child); + if (operand instanceof BinaryPredicate && operand.getOpcode() != TExprOpcode.EQ_FOR_NULL) { + String column = getColumnFromExpr(operand.getChild(0)); + if (column == null) { + column = getColumnFromExpr(operand.getChild(1)); + } + // The complement must still exclude missing scalar values (SQL UNKNOWN). + negated.must(QueryBuilders.existsQuery(fieldsContext.getOrDefault(column, column))); + } + return negated; } default: return null; @@ -157,16 +192,9 @@ private static QueryBuilder parseBinaryPredicate(LiteralExpr expr, TExprOpcode o case EQ_FOR_NULL: return QueryBuilders.termQuery(column, value); case NE: - // col != '' means col.length() > 0 in SQL syntax. - // The `NULL` value should not present in results. - // It equals - // '{"bool":{"must":{"bool":{"must_not":{"term":{"col":""}},"must":{"exists":{"field":"col"}}}}}}' - // in Elasticsearch - if (value instanceof String && StringUtils.isEmpty((String) value)) { - return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(column, value)) + // SQL inequality excludes NULL for every literal, including non-empty values. + return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(column, value)) .must(QueryBuilders.existsQuery(column)); - } - return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(column, value)); case GE: return QueryBuilders.rangeQuery(column).gte(value); case GT: @@ -268,6 +296,40 @@ private static String getColumnFromExpr(Expr expr) { return null; } + public static QueryBuilder toEsDsl(Expr expr, List residuals, Map fieldsContext, + BuilderOptions options, Map column2typeMap, Set unsafeFields) { + // Indexed values can differ from _source; a local residual cannot recover remote false negatives. + if (referencesUnsafeSourceField(expr, unsafeFields)) { + residuals.add(expr); + return null; + } + return toEsDsl(expr, residuals, fieldsContext, options, column2typeMap); + } + + private static boolean isEsQuery(Expr expr) { + return expr instanceof FunctionCallExpr + && "esquery".equalsIgnoreCase(((FunctionCallExpr) expr).getFnName().getFunction()); + } + + private static boolean referencesUnsafeSourceField(Expr expr, Set unsafeFields) { + if (isEsQuery(expr)) { + return false; + } + if (expr instanceof SlotRef) { + return unsafeFields.contains(((SlotRef) expr).getColumnName()); + } + return expr.getChildren().stream().anyMatch(child -> referencesUnsafeSourceField(child, unsafeFields)); + } + + public static void requireLocalEvaluation(Expr expr) { + // Native ES DSL has no BE evaluator and cannot survive in an indivisible residual OR/NOT. + if (isEsQuery(expr)) { + throw new DorisEsException("Cannot evaluate esquery locally: its containing Boolean expression " + + "cannot be fully pushed down to Elasticsearch"); + } + expr.getChildren().forEach(QueryBuilders::requireLocalEvaluation); + } + /** * Doris expr to es dsl. **/ @@ -291,7 +353,7 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map 1 ? expr.getChild(1) : null; column = getColumnFromExpr(rightExpr); opCode = flipOpCode(opCode); isFlip = true; @@ -354,6 +416,7 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map userId.keyword - // this is activated when `enable_keyword_sniff = true` - private Map fetchFieldsContext = Maps.newHashMap(); - // used to indicate which fields can get from ES docavalue - // because elasticsearch can have "fields" feature, field can have - // two or more types, the first type maybe have not docvalue but other - // can have, such as (text field not have docvalue, but keyword can have): - // "properties": { - // "city": { - // "type": "text", - // "fields": { - // "raw": { - // "type": "keyword" - // } - // } - // } - // } - // then the docvalue context provided the mapping between the select field and real request field : - // {"city": "city.raw"} - // use select city from table, if enable the docvalue, we will fetch the `city` field value from `city.raw` - // fetch field value from doc_values, this is activated when `enable_docvalue_scan= true` - private Map docValueFieldsContext = Maps.newHashMap(); - - private List needCompatDateFields = Lists.newArrayList(); + private EsFieldContext fieldContext = new EsFieldContext(); // sourceIndex is the name of index when creating ES external table private final String sourceIndex; @@ -121,15 +96,23 @@ public EsTable esTable() { } public Map fetchFieldsContext() { - return fetchFieldsContext; + return fieldContext.getFetchFieldsContext(); } public Map docValueFieldsContext() { - return docValueFieldsContext; + return fieldContext.getDocValueFieldsContext(); } public List needCompatDateFields() { - return needCompatDateFields; + return fieldContext.getNeedCompatDateFields(); + } + + public EsFieldContext fieldContext() { + return fieldContext; + } + + public void setFieldContext(EsFieldContext fieldContext) { + this.fieldContext = fieldContext; } public void version(EsMajorVersion version) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index ee322221ee7480..c486dd9ab77d93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -17,8 +17,11 @@ package org.apache.doris.datasource.es.source; +import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; @@ -30,6 +33,7 @@ import org.apache.doris.datasource.ExternalScanNode; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.es.EsExternalTable; +import org.apache.doris.datasource.es.EsFieldContext; import org.apache.doris.datasource.es.EsShardPartitions; import org.apache.doris.datasource.es.EsShardRouting; import org.apache.doris.datasource.es.EsTablePartitions; @@ -73,6 +77,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -90,6 +95,7 @@ public class EsScanNode extends ExternalScanNode { private final boolean useFileScannerV2; private QueryBuilder queryBuilder; private boolean isFinalized = false; + private Integer docValuesMode; /** * For multicatalog es. @@ -187,7 +193,11 @@ private TEsScanNode buildEsScanNode() { TEsScanNode esScanNode = new TEsScanNode(desc.getId().asInt()); if (table.isEnableDocValueScan()) { esScanNode.setDocvalueContext(table.docValueContext()); - properties.put(EsResource.DOC_VALUES_MODE, String.valueOf(useDocValueScan(desc, table.docValueContext()))); + properties.put(EsResource.DOC_VALUES_MODE, String.valueOf(docValuesMode == null + ? useDocValueScan(desc, table.docValueContext()) : docValuesMode)); + } + if ((docValuesMode == null ? 0 : docValuesMode) == 0 && table.fieldContext().requiresFullSourceRead()) { + properties.put("read_full_source", "true"); } properties.put(EsResource.QUERY_DSL, queryBuilder.toJson()); if (table.isEnableKeywordSniff() && table.fieldsContext().size() > 0) { @@ -286,13 +296,16 @@ private List getShardLocations() throws UserException { String.join(",", unPartitionedIndices), String.join(",", partitionedIndices)); } List result = Lists.newArrayList(); - boolean enableESParallelScroll = isEnableESParallelScroll(); + // Physical shard requests bypass alias filters/routing. Keep the original alias in one scroll. + boolean aliasScan = selectedIndex.stream().flatMap(index -> index.getShardRoutings().values().stream()) + .flatMap(Collection::stream).anyMatch(shard -> !table.getIndexName().equals(shard.getIndexName())); + boolean enableESParallelScroll = isEnableESParallelScroll() && !aliasScan; for (EsShardPartitions indexState : selectedIndex) { // When disabling parallel scroll, only use the first shard routing. // Because we only need plan a single scan range. List> shardRoutings = enableESParallelScroll ? new ArrayList<>(indexState.getShardRoutings().values()) : - Collections.singletonList(indexState.getShardRoutings().get(0)); + Collections.singletonList(indexState.getShardRoutings().values().iterator().next()); for (List shardRouting : shardRoutings) { // get backends @@ -326,7 +339,8 @@ private List getShardLocations() throws UserException { // such as `log-20240501-1`/`log-20240501-2`. // When we plan a single query, we should use the index alias instead of the real indices names. esScanRange.setIndex( - enableESParallelScroll ? shardRouting.get(0).getIndexName() : indexState.getIndexName()); + enableESParallelScroll ? shardRouting.get(0).getIndexName() + : aliasScan ? table.getIndexName() : indexState.getIndexName()); if (table.getType() != null) { esScanRange.setType(table.getMappingType()); } @@ -338,6 +352,9 @@ private List getShardLocations() throws UserException { locations.setScanRange(scanRange); // result result.add(locations); + if (aliasScan) { + return result; + } } } @@ -418,6 +435,21 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } private void buildQuery() throws UserException { + // Decide scan-wide materialization before considering any remote predicate. + docValuesMode = table.isEnableDocValueScan() ? useDocValueScan(desc, table.docValueContext()) : 0; + EsFieldContext fields = table.fieldContext(); + Set unsafeFields = new HashSet<>(fields.getSourceValueUnsafeFields()); + if (!table.isEnableKeywordSniff()) { + unsafeFields.addAll(fields.getFetchFieldsContext().keySet()); + } + if (docValuesMode == 0) { + unsafeFields.addAll(fields.getSourceModeUnsafeFields()); + conjuncts.forEach(expr -> collectSourceUnsafeFields(expr, unsafeFields)); + } + List flattened = new ArrayList<>(); + conjuncts.forEach(expr -> splitConjuncts(expr, flattened)); + conjuncts.clear(); + conjuncts.addAll(flattened); if (conjuncts.isEmpty()) { queryBuilder = QueryBuilders.matchAllQuery(); } else { @@ -437,10 +469,11 @@ private void buildQuery() throws UserException { QueryBuilder queryBuilder = QueryBuilders.toEsDsl(expr, notPushDownList, fieldsContext, BuilderOptions.builder().likePushDown(table.isLikePushDown()) .needCompatDateFields(table.needCompatDateFields()).build(), - table.getColumn2typeMap()); + table.getColumn2typeMap(), unsafeFields); // A compound predicate is indivisible here: retaining only its unsupported child // would let OR/NOT lose local filtering or push an unsafe partial negation. if (queryBuilder == null || notPushDownList.size() != residualCount) { + QueryBuilders.requireLocalEvaluation(expr); notPushDownList.add(expr); continue; } @@ -456,4 +489,30 @@ private void buildQuery() throws UserException { } } + private static void splitConjuncts(Expr expr, List conjuncts) { + if (expr instanceof CompoundPredicate + && ((CompoundPredicate) expr).getOp() == CompoundPredicate.Operator.AND) { + expr.getChildren().forEach(child -> splitConjuncts(child, conjuncts)); + } else { + conjuncts.add(expr); + } + } + + private void collectSourceUnsafeFields(Expr expr, Set fields) { + if (expr instanceof FunctionCallExpr + && "esquery".equalsIgnoreCase(((FunctionCallExpr) expr).getFnName().getFunction())) { + return; + } + if (expr instanceof SlotRef) { + String name = ((SlotRef) expr).getColumnName(); + if (!"_id".equals(name) && table.getColumn(name) != null + && (table.getColumn(name).getType().isStringType() + || table.getColumn(name).getType().isArrayType())) { + // A source string can be the whole JSON array instead of one indexed element. + fields.add(name); + } + } + expr.getChildren().forEach(child -> collectSourceUnsafeFields(child, fields)); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java new file mode 100644 index 00000000000000..a96e9f7f519827 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EsMappingUtilsTest { + + // ES 6.x style mapping with explicit mapping type "doc" + private static final String MAPPING_ES6 = "{" + + "\"test\": {" + + " \"mappings\": {" + + " \"doc\": {" + + " \"properties\": {" + + " \"k1\": { \"type\": \"long\" }," + + " \"k2\": { \"type\": \"keyword\" }," + + " \"k3\": { \"type\": \"text\", \"fields\": {" + + " \"keyword\": { \"type\": \"keyword\" }" + + " }}," + + " \"k4\": { \"properties\": {" + + " \"child1\": { \"type\": \"text\" }," + + " \"child2\": { \"type\": \"long\" }" + + " }}," + + " \"k5\": { \"type\": \"date\" }," + + " \"k6\": { \"type\": \"nested\", \"properties\": {" + + " \"child3\": { \"type\": \"keyword\" }" + + " }}" + + " }" + + " }" + + " }" + + "}" + + "}"; + + // ES 7+/8+ style mapping without explicit mapping type + private static final String MAPPING_ES7 = "{" + + "\"test\": {" + + " \"mappings\": {" + + " \"properties\": {" + + " \"name\": { \"type\": \"keyword\" }," + + " \"age\": { \"type\": \"integer\" }," + + " \"bio\": { \"type\": \"text\", \"fields\": {" + + " \"raw\": { \"type\": \"keyword\" }" + + " }}," + + " \"created\": { \"type\": \"date\" }," + + " \"addr\": { \"properties\": {" + + " \"city\": { \"type\": \"keyword\" }" + + " }}" + + " }" + + " }" + + "}" + + "}"; + + @Test + public void testColumn2typeMapWithSpecificColumns() { + List columns = Arrays.asList("k1", "k2", "k3", "k5"); + EsFieldContext ctx = EsMappingUtils.resolveFieldContext(columns, "test", MAPPING_ES6, "doc"); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertEquals(4, typeMap.size()); + Assertions.assertEquals("long", typeMap.get("k1")); + Assertions.assertEquals("keyword", typeMap.get("k2")); + Assertions.assertEquals("text", typeMap.get("k3")); + Assertions.assertEquals("date", typeMap.get("k5")); + } + + @Test + public void testColumn2typeMapObjectFieldsWithoutType() { + List columns = Arrays.asList("k4"); + EsFieldContext ctx = EsMappingUtils.resolveFieldContext(columns, "test", MAPPING_ES6, "doc"); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertEquals(1, typeMap.size()); + Assertions.assertEquals("object", typeMap.get("k4")); + } + + @Test + public void testColumn2typeMapPopulatedWhenColumnNamesEmpty() { + // When columnNames is empty, all mapping properties should be resolved + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + Collections.emptyList(), "test", MAPPING_ES6, "doc"); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertEquals(6, typeMap.size()); + Assertions.assertEquals("long", typeMap.get("k1")); + Assertions.assertEquals("keyword", typeMap.get("k2")); + Assertions.assertEquals("text", typeMap.get("k3")); + Assertions.assertEquals("object", typeMap.get("k4")); + Assertions.assertEquals("date", typeMap.get("k5")); + Assertions.assertEquals("nested", typeMap.get("k6")); + } + + @Test + public void testColumn2typeMapPopulatedWhenColumnNamesNull() { + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + null, "test", MAPPING_ES6, "doc"); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertFalse(typeMap.isEmpty()); + Assertions.assertEquals("long", typeMap.get("k1")); + } + + @Test + public void testFieldContextKeywordSniffWhenColumnsEmpty() { + // k3 is "text" with a "keyword" sub-field — should be detected + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + Collections.emptyList(), "test", MAPPING_ES6, "doc"); + + Map fetchFields = ctx.getFetchFieldsContext(); + Assertions.assertTrue(fetchFields.containsKey("k3")); + Assertions.assertEquals("k3.keyword", fetchFields.get("k3")); + } + + @Test + public void testFieldContextDateCompatWhenColumnsEmpty() { + // k5 is "date" with no format → needs compat + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + Collections.emptyList(), "test", MAPPING_ES6, "doc"); + + List dateFields = ctx.getNeedCompatDateFields(); + Assertions.assertTrue(dateFields.contains("k5")); + } + + @Test + public void testColumn2typeMapEs7Mapping() { + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + Collections.emptyList(), "test", MAPPING_ES7, null); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertEquals(5, typeMap.size()); + Assertions.assertEquals("keyword", typeMap.get("name")); + Assertions.assertEquals("integer", typeMap.get("age")); + Assertions.assertEquals("text", typeMap.get("bio")); + Assertions.assertEquals("date", typeMap.get("created")); + Assertions.assertEquals("object", typeMap.get("addr")); + } + + @Test + public void testEs7KeywordSniffAndDateCompat() { + List columns = Arrays.asList("bio", "created"); + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + columns, "test", MAPPING_ES7, null); + + // bio is text with keyword sub-field "raw" + Assertions.assertEquals("bio.raw", ctx.getFetchFieldsContext().get("bio")); + // created is date with no format + Assertions.assertTrue(ctx.getNeedCompatDateFields().contains("created")); + } + + @Test + public void testIdColumnSkipped() { + List columns = Arrays.asList("_id", "k1"); + EsFieldContext ctx = EsMappingUtils.resolveFieldContext( + columns, "test", MAPPING_ES6, "doc"); + + Map typeMap = ctx.getColumn2typeMap(); + Assertions.assertEquals(1, typeMap.size()); + Assertions.assertFalse(typeMap.containsKey("_id")); + Assertions.assertEquals("long", typeMap.get("k1")); + } + + @Test + public void legacyTypeNamesCanResembleRootParameters() { + for (String type : Arrays.asList("doc", "_doc", "enabled", "date_detection", "numeric_detection", + "dynamic_date_formats", "runtime", "custom_properties")) { + String mapping = "{\"test\":{\"mappings\":{\"" + type + + "\":{\"properties\":{\"a\":{\"type\":\"long\"}}}}}}"; + Assertions.assertEquals(org.apache.doris.catalog.Type.BIGINT, EsUtil.genColumnsFromEs( + "test", null, EsMappingUtils.getRootSchema(EsMappingUtils.getMapping(mapping), null, + new ArrayList<>()), false, new ArrayList<>(), new HashMap<>()).get(0).getType(), type); + Assertions.assertEquals("long", EsMappingUtils.resolveFieldContext( + Collections.singletonList("a"), "test", mapping, type).getColumn2typeMap().get("a"), type); + } + } + + @Test + void runtimeFieldNamedPropertiesIsNotALegacyType() throws Exception { + ObjectNode mapping = (ObjectNode) new ObjectMapper().readTree( + "{\"runtime\":{\"properties\":{\"type\":\"long\"}}}"); + ObjectNode root = EsMappingUtils.getRootSchema(mapping, null, new ArrayList<>()); + Assertions.assertTrue(root.has("runtime")); + Assertions.assertEquals("long", root.path("runtime").path("properties").path("type").asText()); + } + + @Test + void indexMalformedDefaultsRespectFieldOverridesAndFieldTypes() { + String mapping = "{\"test\":{\"settings\":{\"index\":{\"mapping\":" + + "{\"ignore_malformed\":\"true\",\"coerce\":\"false\"}}},\"mappings\":{\"properties\":{" + + "\"inherited\":{\"type\":\"long\"}," + + "\"overridden\":{\"type\":\"long\",\"ignore_malformed\":false}," + + "\"keyword\":{\"type\":\"keyword\"}}}}}"; + EsFieldContext context = EsMappingUtils.resolveFieldContext(Collections.emptyList(), "test", mapping, null); + Assertions.assertTrue(context.getSourceValueUnsafeFields().contains("inherited")); + Assertions.assertFalse(context.getDocValueFieldsContext().containsKey("inherited")); + Assertions.assertFalse(context.getSourceValueUnsafeFields().contains("overridden")); + Assertions.assertTrue(context.getDocValueFieldsContext().containsKey("overridden")); + Assertions.assertFalse(context.getSourceValueUnsafeFields().contains("keyword")); + Assertions.assertTrue(context.getDocValueFieldsContext().containsKey("keyword")); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsRestClientVersionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsRestClientVersionTest.java index 2e12383d818082..d93a56f034a11e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsRestClientVersionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsRestClientVersionTest.java @@ -27,6 +27,32 @@ import java.util.Map; class EsRestClientVersionTest { + @Test + void mappingRequestIncludesIndexDefaults() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + java.util.concurrent.atomic.AtomicReference query = new java.util.concurrent.atomic.AtomicReference<>(); + String response = "{\"sample\":{\"mappings\":{\"properties\":{\"a\":{\"type\":\"integer\"}}}," + + "\"settings\":{\"index\":{\"mapping\":{\"ignore_malformed\":\"true\"}}}}}"; + server.createContext("/sample", exchange -> { + query.set(exchange.getRequestURI().getQuery()); + byte[] body = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + EsRestClient client = new EsRestClient(new String[]{"http://127.0.0.1:" + + server.getAddress().getPort()}, null, null, false); + Assertions.assertEquals(response, client.getMapping("sample")); + Assertions.assertEquals("filter_path=*.mappings,*.settings.index.mapping.*", query.get()); + Assertions.assertTrue(EsMappingUtils.resolveFieldContext(java.util.Collections.singletonList("a"), + "sample", response, null).getSourceValueUnsafeFields().contains("a")); + } finally { + server.stop(0); + } + } + @Test void mixedEightAndNineNodesAreDiscoveredDuringUpgrade() throws Exception { String nodes = "{\"nodes\":{" diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java index c6bba4cbc260e5..dd8cec09e415fd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java @@ -25,16 +25,24 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.FederationBackendPolicy; +import org.apache.doris.datasource.es.DorisEsException; +import org.apache.doris.datasource.es.EsFieldContext; +import org.apache.doris.datasource.es.EsShardPartitions; +import org.apache.doris.datasource.es.EsShardRouting; +import org.apache.doris.datasource.es.EsTablePartitions; import org.apache.doris.datasource.es.QueryBuilders; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; import org.apache.doris.thrift.TEsScanRange; import org.apache.doris.thrift.TFileScanRange; import org.apache.doris.thrift.TNetworkAddress; @@ -48,6 +56,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import java.lang.reflect.Field; @@ -155,6 +164,108 @@ void keywordAliasesSurviveFileScanThriftPopulation() throws Exception { Assertions.assertEquals("{\"match_all\":{}}", params.getEsProperties().get(EsResource.QUERY_DSL)); } + @Test + void sourceStringsAndNativeResidualsAreHandledBeforePushdown() throws Exception { + for (boolean v2 : Arrays.asList(false, true)) { + TestScanNode node = createNode(v2); + EsTable table = (EsTable) node.getTupleDesc().getTable(); + Column name = new Column("name", Type.STRING); + Mockito.doReturn(name).when(table).getColumn("name"); + Expr local = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "name"), new StringLiteral("one")); + Expr nativeQuery = new FunctionCallExpr("esquery", new ArrayList<>(Arrays.asList( + new SlotRef(null, "id"), new StringLiteral("{\"term\":{\"id\":1}}")))); + node.getConjuncts().add(new CompoundPredicate(CompoundPredicate.Operator.AND, local, nativeQuery)); + Method build = EsScanNode.class.getDeclaredMethod("buildQuery"); + build.setAccessible(true); + build.invoke(node); + Assertions.assertEquals(Collections.singletonList(local), node.getConjuncts()); + Assertions.assertTrue(node.getNodeExplainString("", org.apache.doris.thrift.TExplainLevel.NORMAL) + .contains("\"id\":1")); + TestScanNode rejected = createNode(v2); + EsTable other = (EsTable) rejected.getTupleDesc().getTable(); + Mockito.doReturn(name).when(other).getColumn("name"); + rejected.getConjuncts().add(new CompoundPredicate(CompoundPredicate.Operator.OR, local, nativeQuery)); + java.lang.reflect.InvocationTargetException error = Assertions.assertThrows( + java.lang.reflect.InvocationTargetException.class, () -> build.invoke(rejected)); + Assertions.assertInstanceOf(DorisEsException.class, error.getCause()); + Assertions.assertTrue(error.getCause().getMessage().contains("Cannot evaluate esquery locally")); + } + } + + @Test + void finalReadModeControlsPredicateSafetyAndFullSourceRequest() throws Exception { + for (boolean docvalues : Arrays.asList(false, true)) { + TestScanNode node = createNode(true); + EsTable table = (EsTable) node.getTupleDesc().getTable(); + Mockito.doReturn(docvalues).when(table).isEnableDocValueScan(); + Mockito.doReturn(ImmutableMap.of("id", "id")).when(table).docValueContext(); + EsFieldContext fields = new EsFieldContext(); + fields.getSourceModeUnsafeFields().add("id"); + fields.requireFullSourceRead(); + Mockito.doReturn(fields).when(table).fieldContext(); + Expr predicate = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "id"), new IntLiteral(1)); + node.getConjuncts().add(predicate); + Method build = EsScanNode.class.getDeclaredMethod("buildQuery"); + build.setAccessible(true); + build.invoke(node); + Assertions.assertEquals(docvalues ? Collections.emptyList() : Collections.singletonList(predicate), + node.getConjuncts()); + java.util.Map properties = node.getScanRangeLocations(0).get(0).getScanRange() + .getExtScanRange().getFileScanRange().getParams().getEsProperties(); + Assertions.assertEquals(!docvalues, properties.containsKey("read_full_source")); + Assertions.assertEquals(docvalues ? "1" : null, properties.get(EsResource.DOC_VALUES_MODE)); + } + } + + @Test + void aliasesKeepTheirOriginalNameAndOneCoordinatingScroll() throws Exception { + for (boolean alias : Arrays.asList(false, true)) { + for (boolean parallel : Arrays.asList(false, true)) { + TestScanNode node = createNode(false); + ConnectContext.get().getSessionVariable().setEnableEsParallelScroll(parallel); + EsTable table = (EsTable) node.getTupleDesc().getTable(); + Mockito.doReturn(alias ? "filtered_alias" : "es_index").when(table).getIndexName(); + EsShardRouting first = new EsShardRouting("es_index", 3, true, "node"); + first.setHttpAddress(new TNetworkAddress("127.0.0.1", 9200)); + EsShardRouting second = new EsShardRouting("es_index", 5, true, "node"); + second.setHttpAddress(first.getHttpAddress()); + EsShardPartitions index = Mockito.mock(EsShardPartitions.class); + Mockito.when(index.getIndexName()).thenReturn(alias ? "filtered_alias" : "es_index"); + Mockito.when(index.getShardRoutings()).thenReturn(ImmutableMap.of( + 3, Collections.singletonList(first), 5, Collections.singletonList(second))); + EsTablePartitions partitions = new EsTablePartitions(); + partitions.addIndexState(table.getIndexName(), index); + Field field = EsScanNode.class.getDeclaredField("esTablePartitions"); + field.setAccessible(true); + field.set(node, partitions); + Backend backend = Mockito.mock(Backend.class); + Mockito.when(backend.getHost()).thenReturn("127.0.0.1"); + try (MockedConstruction policies = Mockito.mockConstruction( + FederationBackendPolicy.class, (policy, context) -> { + Mockito.when(policy.numBackends()).thenReturn(1); + Mockito.when(policy.getNextBe()).thenReturn(backend); + })) { + Method method = EsScanNode.class.getDeclaredMethod("getShardLocations"); + method.setAccessible(true); + @SuppressWarnings("unchecked") + List ranges = (List) method.invoke(node); + Assertions.assertEquals(alias || !parallel ? 1 : 2, ranges.size()); + for (TScanRangeLocations range : ranges) { + TEsScanRange es = range.getScanRange().getEsScanRange(); + Assertions.assertEquals(table.getIndexName(), es.getIndex()); + if (alias || !parallel) { + Assertions.assertEquals(-1, es.getShardId()); + } else { + Assertions.assertTrue(es.getShardId() == 3 || es.getShardId() == 5); + } + } + } + } + } + } + private org.apache.doris.thrift.TFileScanRangeParams projectionParams( List names, int maxFields, boolean docvalues) throws Exception { ConnectContext context = new ConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index fe4ccb9a8245ef..a94589bdf7c685 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -60,6 +60,27 @@ public class QueryBuildersTest { private final ObjectMapper mapper = new ObjectMapper(); + @Test + public void retainedNegationsPreserveMissingAndArraySemantics() throws Exception { + Expr equality = new BinaryPredicate(Operator.EQ, new SlotRef(null, "a"), new IntLiteral(2)); + Expr notEqual = new CompoundPredicate(CompoundPredicate.Operator.NOT, equality, null); + Assertions.assertTrue(mapper.readTree(QueryBuilders.toEsDsl(notEqual, Collections.emptyMap()).toJson()) + .path("bool").has("must")); + Expr doubleNot = new CompoundPredicate(CompoundPredicate.Operator.NOT, notEqual, null); + Assertions.assertEquals(QueryBuilders.toEsDsl(equality, Collections.emptyMap()).toJson(), + QueryBuilders.toEsDsl(doubleNot, Collections.emptyMap()).toJson()); + Expr range = new BinaryPredicate(Operator.LT, new SlotRef(null, "a"), new IntLiteral(2)); + Expr notRange = new CompoundPredicate(CompoundPredicate.Operator.NOT, range, null); + com.fasterxml.jackson.databind.JsonNode dsl = mapper.readTree( + QueryBuilders.toEsDsl(notRange, Collections.emptyMap()).toJson()).path("bool"); + Assertions.assertEquals(2, dsl.path("must_not").path("range").path("a").path("lt").asInt()); + Assertions.assertEquals("a", dsl.path("must").path("exists").path("field").asText()); + Expr nullSafe = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "a"), new IntLiteral(2)); + Assertions.assertFalse(mapper.readTree(QueryBuilders.toEsDsl(new CompoundPredicate( + CompoundPredicate.Operator.NOT, nullSafe, null), Collections.emptyMap()).toJson()) + .path("bool").has("must")); + } + @Test public void nestedResidualOrAndNotRemainLocal() { Expr supported = new BinaryPredicate(Operator.EQ, new SlotRef(null, "name"), new StringLiteral("one")); @@ -89,7 +110,7 @@ public void testBinaryPredicateConvertEsDsl() { Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3)); Map column2typeMap = new HashMap<>(); Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr, column2typeMap).toJson()); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", + Assertions.assertEquals("{\"bool\":{\"must\":{\"exists\":{\"field\":\"k1\"}},\"must_not\":{\"term\":{\"k1\":3}}}}", QueryBuilders.toEsDsl(neExpr, column2typeMap).toJson()); Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr, column2typeMap).toJson()); Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr, column2typeMap).toJson()); @@ -111,7 +132,7 @@ public void testBinaryPredicateConvertEsDsl() { QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); stringLiteral = new StringLiteral("message"); stringNeExpr = new BinaryPredicate(Operator.NE, k3, stringLiteral); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k3\":\"message\"}}}}", + Assertions.assertEquals("{\"bool\":{\"must\":{\"exists\":{\"field\":\"k3\"}},\"must_not\":{\"term\":{\"k3\":\"message\"}}}}", QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); } @@ -133,7 +154,7 @@ public void testCompoundPredicateConvertEsDsl() { QueryBuilders.toEsDsl(andPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", QueryBuilders.toEsDsl(orPredicate, column2typeMap).toJson()); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", + Assertions.assertEquals("{\"bool\":{\"must\":{\"exists\":{\"field\":\"k1\"}},\"must_not\":{\"term\":{\"k1\":3}}}}", QueryBuilders.toEsDsl(notPredicate, column2typeMap).toJson()); } diff --git a/regression-test/data/external_table_p2/es/test_es_epoch_millis.out b/regression-test/data/external_table_p2/es/test_es_epoch_millis.out new file mode 100644 index 00000000000000..271026b76f7848 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_epoch_millis.out @@ -0,0 +1,721 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_true_v2_false_parallel_false_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_false_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_false_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_true_v2_false_parallel_false_zone_0_source_3 -- + +-- !dv_true_v2_false_parallel_false_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_true_v2_false_parallel_false_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_false_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_false_zone_8_source_2 -- + +-- !dv_true_v2_false_parallel_false_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_true_v2_false_parallel_false_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_true_v2_false_parallel_true_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_true_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_true_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_true_v2_false_parallel_true_zone_0_source_3 -- + +-- !dv_true_v2_false_parallel_true_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_true_v2_false_parallel_true_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_true_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_false_parallel_true_zone_8_source_2 -- + +-- !dv_true_v2_false_parallel_true_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_true_v2_false_parallel_true_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_true_v2_true_parallel_false_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_false_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_false_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_true_v2_true_parallel_false_zone_0_source_3 -- + +-- !dv_true_v2_true_parallel_false_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_true_v2_true_parallel_false_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_false_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_false_zone_8_source_2 -- + +-- !dv_true_v2_true_parallel_false_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_true_v2_true_parallel_false_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_true_v2_true_parallel_true_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_true_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_true_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_true_v2_true_parallel_true_zone_0_source_3 -- + +-- !dv_true_v2_true_parallel_true_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_true_v2_true_parallel_true_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_true_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_true_v2_true_parallel_true_zone_8_source_2 -- + +-- !dv_true_v2_true_parallel_true_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_true_v2_true_parallel_true_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_false_v2_false_parallel_false_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_false_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_false_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_false_v2_false_parallel_false_zone_0_source_3 -- + +-- !dv_false_v2_false_parallel_false_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_false_v2_false_parallel_false_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_false_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_false_zone_8_source_2 -- + +-- !dv_false_v2_false_parallel_false_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_false_v2_false_parallel_false_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_false_v2_false_parallel_true_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_true_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_true_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_false_v2_false_parallel_true_zone_0_source_3 -- + +-- !dv_false_v2_false_parallel_true_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_false_v2_false_parallel_true_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_true_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_false_parallel_true_zone_8_source_2 -- + +-- !dv_false_v2_false_parallel_true_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_false_v2_false_parallel_true_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_false_v2_true_parallel_false_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_false_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_false_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_false_v2_true_parallel_false_zone_0_source_3 -- + +-- !dv_false_v2_true_parallel_false_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_false_v2_true_parallel_false_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_false_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_false_zone_8_source_2 -- + +-- !dv_false_v2_true_parallel_false_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_false_v2_true_parallel_false_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + +-- !dv_false_v2_true_parallel_true_zone_0_source_0 -- +1 VALUE 1970-01-01T00:00 +2 VALUE 1970-01-01T00:00 +3 VALUE 1970-01-01T00:00 +4 VALUE 1970-01-01T00:00 +5 VALUE 1970-01-01T00:00:01 +6 VALUE 1970-01-01T00:00:12 +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 +10 VALUE 2023-03-04T02:08:48 +11 VALUE \N +12 VALUE 1970-01-01T00:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_true_zone_0_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_true_zone_0_source_2 -- +7 VALUE 1969-12-31T23:59:59 +8 VALUE 1969-12-31T23:59:59 +9 VALUE 1969-12-31T23:59:58 + +-- !dv_false_v2_true_parallel_true_zone_0_source_3 -- + +-- !dv_false_v2_true_parallel_true_zone_0_control -- +1 1970-01-01T00:00 +2 1970-01-01T00:00 +3 1970-01-01T00:00 +4 1970-01-01T00:00 +5 1970-01-01T00:00:01 +6 1970-01-01T00:00:12 +7 1969-12-31T23:59:59 +8 1969-12-31T23:59:59 +9 1969-12-31T23:59:58 +10 2023-03-04T02:08:48 +11 \N +12 1970-01-01T00:00 +13 \N +14 \N + +-- !dv_false_v2_true_parallel_true_zone_8_source_0 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +5 VALUE 1970-01-01T08:00:01 +6 VALUE 1970-01-01T08:00:12 +7 VALUE 1970-01-01T07:59:59 +8 VALUE 1970-01-01T07:59:59 +9 VALUE 1970-01-01T07:59:58 +10 VALUE 2023-03-04T10:08:48 +11 VALUE \N +12 VALUE 1970-01-01T08:00 +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_true_zone_8_source_1 -- +11 VALUE \N +13 VALUE \N +14 VALUE \N + +-- !dv_false_v2_true_parallel_true_zone_8_source_2 -- + +-- !dv_false_v2_true_parallel_true_zone_8_source_3 -- +1 VALUE 1970-01-01T08:00 +2 VALUE 1970-01-01T08:00 +3 VALUE 1970-01-01T08:00 +4 VALUE 1970-01-01T08:00 +12 VALUE 1970-01-01T08:00 + +-- !dv_false_v2_true_parallel_true_zone_8_control -- +1 1970-01-01T08:00 +2 1970-01-01T08:00 +3 1970-01-01T08:00 +4 1970-01-01T08:00 +5 1970-01-01T08:00:01 +6 1970-01-01T08:00:12 +7 1970-01-01T07:59:59 +8 1970-01-01T07:59:59 +9 1970-01-01T07:59:58 +10 2023-03-04T10:08:48 +11 \N +12 1970-01-01T08:00 +13 \N +14 \N + diff --git a/regression-test/data/external_table_p2/es/test_es_mapping_boundaries.out b/regression-test/data/external_table_p2/es/test_es_mapping_boundaries.out new file mode 100644 index 00000000000000..b04b89161e1105 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_mapping_boundaries.out @@ -0,0 +1,1129 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_true_v2_false_parallel_false_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_false_parallel_false_native_1 -- +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_native_2 -- + +-- !dv_true_v2_false_parallel_false_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_native_6 -- +4 \N \N false + +-- !dv_true_v2_false_parallel_false_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_false_parallel_false_native_8 -- +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_false_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_false_alias_filter_0 -- +99 + +-- !dv_true_v2_false_parallel_false_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_false_alias_filter_1 -- +99 + +-- !dv_true_v2_false_parallel_false_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_2 -- + +-- !dv_true_v2_false_parallel_false_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_3 -- + +-- !dv_true_v2_false_parallel_false_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_4 -- + +-- !dv_true_v2_false_parallel_false_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_5 -- + +-- !dv_true_v2_false_parallel_false_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_6 -- + +-- !dv_true_v2_false_parallel_false_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_7 -- + +-- !dv_true_v2_false_parallel_false_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_8 -- + +-- !dv_true_v2_false_parallel_false_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_false_alias_filter_9 -- + +-- !dv_true_v2_false_parallel_false_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_false_alias_filter_10 -- +99 + +-- !dv_true_v2_false_parallel_false_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_false_alias_filter_11 -- +99 + +-- !dv_true_v2_false_parallel_true_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_false_parallel_true_native_1 -- +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_native_2 -- + +-- !dv_true_v2_false_parallel_true_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_native_6 -- +4 \N \N false + +-- !dv_true_v2_false_parallel_true_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_false_parallel_true_native_8 -- +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_false_parallel_true_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_true_alias_filter_0 -- +99 + +-- !dv_true_v2_false_parallel_true_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_true_alias_filter_1 -- +99 + +-- !dv_true_v2_false_parallel_true_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_2 -- + +-- !dv_true_v2_false_parallel_true_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_3 -- + +-- !dv_true_v2_false_parallel_true_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_4 -- + +-- !dv_true_v2_false_parallel_true_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_5 -- + +-- !dv_true_v2_false_parallel_true_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_6 -- + +-- !dv_true_v2_false_parallel_true_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_7 -- + +-- !dv_true_v2_false_parallel_true_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_8 -- + +-- !dv_true_v2_false_parallel_true_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_false_parallel_true_alias_filter_9 -- + +-- !dv_true_v2_false_parallel_true_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_true_alias_filter_10 -- +99 + +-- !dv_true_v2_false_parallel_true_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_false_parallel_true_alias_filter_11 -- +99 + +-- !dv_true_v2_true_parallel_false_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_true_parallel_false_native_1 -- +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_native_2 -- + +-- !dv_true_v2_true_parallel_false_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_native_6 -- +4 \N \N false + +-- !dv_true_v2_true_parallel_false_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_true_parallel_false_native_8 -- +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_false_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_false_alias_filter_0 -- +99 + +-- !dv_true_v2_true_parallel_false_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_false_alias_filter_1 -- +99 + +-- !dv_true_v2_true_parallel_false_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_2 -- + +-- !dv_true_v2_true_parallel_false_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_3 -- + +-- !dv_true_v2_true_parallel_false_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_4 -- + +-- !dv_true_v2_true_parallel_false_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_5 -- + +-- !dv_true_v2_true_parallel_false_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_6 -- + +-- !dv_true_v2_true_parallel_false_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_7 -- + +-- !dv_true_v2_true_parallel_false_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_8 -- + +-- !dv_true_v2_true_parallel_false_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_false_alias_filter_9 -- + +-- !dv_true_v2_true_parallel_false_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_false_alias_filter_10 -- +99 + +-- !dv_true_v2_true_parallel_false_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_false_alias_filter_11 -- +99 + +-- !dv_true_v2_true_parallel_true_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_true_parallel_true_native_1 -- +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_native_2 -- + +-- !dv_true_v2_true_parallel_true_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_native_6 -- +4 \N \N false + +-- !dv_true_v2_true_parallel_true_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_true_v2_true_parallel_true_native_8 -- +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_true_v2_true_parallel_true_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_true_alias_filter_0 -- +99 + +-- !dv_true_v2_true_parallel_true_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_true_alias_filter_1 -- +99 + +-- !dv_true_v2_true_parallel_true_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_2 -- + +-- !dv_true_v2_true_parallel_true_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_3 -- + +-- !dv_true_v2_true_parallel_true_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_4 -- + +-- !dv_true_v2_true_parallel_true_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_5 -- + +-- !dv_true_v2_true_parallel_true_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_6 -- + +-- !dv_true_v2_true_parallel_true_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_7 -- + +-- !dv_true_v2_true_parallel_true_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_8 -- + +-- !dv_true_v2_true_parallel_true_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_true_v2_true_parallel_true_alias_filter_9 -- + +-- !dv_true_v2_true_parallel_true_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_true_alias_filter_10 -- +99 + +-- !dv_true_v2_true_parallel_true_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_true_v2_true_parallel_true_alias_filter_11 -- +99 + +-- !dv_false_v2_false_parallel_false_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_false_parallel_false_native_1 -- +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_native_2 -- + +-- !dv_false_v2_false_parallel_false_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_native_6 -- +4 \N \N false + +-- !dv_false_v2_false_parallel_false_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_false_parallel_false_native_8 -- +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_false_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_false_alias_filter_0 -- +99 + +-- !dv_false_v2_false_parallel_false_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_false_alias_filter_1 -- +99 + +-- !dv_false_v2_false_parallel_false_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_2 -- + +-- !dv_false_v2_false_parallel_false_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_3 -- + +-- !dv_false_v2_false_parallel_false_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_4 -- + +-- !dv_false_v2_false_parallel_false_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_5 -- + +-- !dv_false_v2_false_parallel_false_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_6 -- + +-- !dv_false_v2_false_parallel_false_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_7 -- + +-- !dv_false_v2_false_parallel_false_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_8 -- + +-- !dv_false_v2_false_parallel_false_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_false_alias_filter_9 -- + +-- !dv_false_v2_false_parallel_false_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_false_alias_filter_10 -- +99 + +-- !dv_false_v2_false_parallel_false_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_false_alias_filter_11 -- +99 + +-- !dv_false_v2_false_parallel_true_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_false_parallel_true_native_1 -- +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_native_2 -- + +-- !dv_false_v2_false_parallel_true_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_native_6 -- +4 \N \N false + +-- !dv_false_v2_false_parallel_true_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_false_parallel_true_native_8 -- +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_false_parallel_true_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_true_alias_filter_0 -- +99 + +-- !dv_false_v2_false_parallel_true_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_true_alias_filter_1 -- +99 + +-- !dv_false_v2_false_parallel_true_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_2 -- + +-- !dv_false_v2_false_parallel_true_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_3 -- + +-- !dv_false_v2_false_parallel_true_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_4 -- + +-- !dv_false_v2_false_parallel_true_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_5 -- + +-- !dv_false_v2_false_parallel_true_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_6 -- + +-- !dv_false_v2_false_parallel_true_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_7 -- + +-- !dv_false_v2_false_parallel_true_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_8 -- + +-- !dv_false_v2_false_parallel_true_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_false_parallel_true_alias_filter_9 -- + +-- !dv_false_v2_false_parallel_true_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_true_alias_filter_10 -- +99 + +-- !dv_false_v2_false_parallel_true_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_false_parallel_true_alias_filter_11 -- +99 + +-- !dv_false_v2_true_parallel_false_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_true_parallel_false_native_1 -- +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_native_2 -- + +-- !dv_false_v2_true_parallel_false_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_native_6 -- +4 \N \N false + +-- !dv_false_v2_true_parallel_false_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_true_parallel_false_native_8 -- +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_false_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_false_alias_filter_0 -- +99 + +-- !dv_false_v2_true_parallel_false_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_false_alias_filter_1 -- +99 + +-- !dv_false_v2_true_parallel_false_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_2 -- + +-- !dv_false_v2_true_parallel_false_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_3 -- + +-- !dv_false_v2_true_parallel_false_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_4 -- + +-- !dv_false_v2_true_parallel_false_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_5 -- + +-- !dv_false_v2_true_parallel_false_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_6 -- + +-- !dv_false_v2_true_parallel_false_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_7 -- + +-- !dv_false_v2_true_parallel_false_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_8 -- + +-- !dv_false_v2_true_parallel_false_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_false_alias_filter_9 -- + +-- !dv_false_v2_true_parallel_false_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_false_alias_filter_10 -- +99 + +-- !dv_false_v2_true_parallel_false_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_false_alias_filter_11 -- +99 + +-- !dv_false_v2_true_parallel_true_native_0 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_true_parallel_true_native_1 -- +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_native_2 -- + +-- !dv_false_v2_true_parallel_true_native_3 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_native_4 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_native_5 -- +1 1 1700000000000 true +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_native_6 -- +4 \N \N false + +-- !dv_false_v2_true_parallel_true_native_7 -- +1 1 1700000000000 true +2 2 1700000001000 false + +-- !dv_false_v2_true_parallel_true_native_8 -- +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_native_9 -- +2 2 1700000001000 false +3 3 1700000002000 true + +-- !dv_false_v2_true_parallel_true_alias_0 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_true_alias_filter_0 -- +99 + +-- !dv_false_v2_true_parallel_true_alias_1 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_true_alias_filter_1 -- +99 + +-- !dv_false_v2_true_parallel_true_alias_2 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_2 -- + +-- !dv_false_v2_true_parallel_true_alias_3 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_3 -- + +-- !dv_false_v2_true_parallel_true_alias_4 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_4 -- + +-- !dv_false_v2_true_parallel_true_alias_5 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_5 -- + +-- !dv_false_v2_true_parallel_true_alias_6 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_6 -- + +-- !dv_false_v2_true_parallel_true_alias_7 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_7 -- + +-- !dv_false_v2_true_parallel_true_alias_8 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_8 -- + +-- !dv_false_v2_true_parallel_true_alias_9 -- +1 1 +2 2 +3 3 +4 \N + +-- !dv_false_v2_true_parallel_true_alias_filter_9 -- + +-- !dv_false_v2_true_parallel_true_alias_10 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_true_alias_filter_10 -- +99 + +-- !dv_false_v2_true_parallel_true_alias_11 -- +1 1 +2 2 +3 3 +4 \N +99 99 + +-- !dv_false_v2_true_parallel_true_alias_filter_11 -- +99 + diff --git a/regression-test/data/external_table_p2/es/test_es_numeric_source_semantics.out b/regression-test/data/external_table_p2/es/test_es_numeric_source_semantics.out new file mode 100644 index 00000000000000..54363f1a93c2c4 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_numeric_source_semantics.out @@ -0,0 +1,813 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !enabled_v2_false_parallel_false_predicate_0 -- + +-- !enabled_v2_false_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_8 -- +2 empty \N \N + +-- !enabled_v2_false_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_false_predicate_13 -- + +-- !enabled_v2_false_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_false_filter_only -- + +-- !enabled_v2_false_parallel_false_native -- +ok 2.34 + +-- !enabled_v2_false_parallel_false_native_and -- + +-- !enabled_v2_false_parallel_false_b_docvalue_control -- +2.25 + +-- !enabled_v2_false_parallel_false_c_docvalue_control -- +2.3398438 + +-- !enabled_v2_false_parallel_false_numeric_projection -- +1 2.25 2.3398438 + +-- !enabled_v2_false_parallel_true_predicate_0 -- + +-- !enabled_v2_false_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_8 -- +2 empty \N \N + +-- !enabled_v2_false_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_true_predicate_13 -- + +-- !enabled_v2_false_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !enabled_v2_false_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_false_parallel_true_filter_only -- + +-- !enabled_v2_false_parallel_true_native -- +ok 2.34 + +-- !enabled_v2_false_parallel_true_native_and -- + +-- !enabled_v2_false_parallel_true_b_docvalue_control -- +2.25 + +-- !enabled_v2_false_parallel_true_c_docvalue_control -- +2.3398438 + +-- !enabled_v2_false_parallel_true_numeric_projection -- +1 2.25 2.3398438 + +-- !enabled_v2_true_parallel_false_predicate_0 -- + +-- !enabled_v2_true_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_8 -- +2 empty \N \N + +-- !enabled_v2_true_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_false_predicate_13 -- + +-- !enabled_v2_true_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_false_filter_only -- + +-- !enabled_v2_true_parallel_false_native -- +ok 2.34 + +-- !enabled_v2_true_parallel_false_native_and -- + +-- !enabled_v2_true_parallel_false_b_docvalue_control -- +2.25 + +-- !enabled_v2_true_parallel_false_c_docvalue_control -- +2.3398438 + +-- !enabled_v2_true_parallel_false_numeric_projection -- +1 2.25 2.3398438 + +-- !enabled_v2_true_parallel_true_predicate_0 -- + +-- !enabled_v2_true_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_8 -- +2 empty \N \N + +-- !enabled_v2_true_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_true_predicate_13 -- + +-- !enabled_v2_true_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !enabled_v2_true_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !enabled_v2_true_parallel_true_filter_only -- + +-- !enabled_v2_true_parallel_true_native -- +ok 2.34 + +-- !enabled_v2_true_parallel_true_native_and -- + +-- !enabled_v2_true_parallel_true_b_docvalue_control -- +2.25 + +-- !enabled_v2_true_parallel_true_c_docvalue_control -- +2.3398438 + +-- !enabled_v2_true_parallel_true_numeric_projection -- +1 2.25 2.3398438 + +-- !disabled_v2_false_parallel_false_predicate_0 -- + +-- !disabled_v2_false_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_8 -- +2 empty \N \N + +-- !disabled_v2_false_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_false_predicate_13 -- + +-- !disabled_v2_false_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_false_filter_only -- + +-- !disabled_v2_false_parallel_false_native -- +ok 2.34 + +-- !disabled_v2_false_parallel_false_native_and -- + +-- !disabled_v2_false_parallel_false_b_docvalue_control -- + +-- !disabled_v2_false_parallel_false_c_docvalue_control -- + +-- !disabled_v2_false_parallel_false_numeric_projection -- + +-- !disabled_v2_false_parallel_true_predicate_0 -- + +-- !disabled_v2_false_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_8 -- +2 empty \N \N + +-- !disabled_v2_false_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_true_predicate_13 -- + +-- !disabled_v2_false_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !disabled_v2_false_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_false_parallel_true_filter_only -- + +-- !disabled_v2_false_parallel_true_native -- +ok 2.34 + +-- !disabled_v2_false_parallel_true_native_and -- + +-- !disabled_v2_false_parallel_true_b_docvalue_control -- + +-- !disabled_v2_false_parallel_true_c_docvalue_control -- + +-- !disabled_v2_false_parallel_true_numeric_projection -- + +-- !disabled_v2_true_parallel_false_predicate_0 -- + +-- !disabled_v2_true_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_8 -- +2 empty \N \N + +-- !disabled_v2_true_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_false_predicate_13 -- + +-- !disabled_v2_true_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_false_filter_only -- + +-- !disabled_v2_true_parallel_false_native -- +ok 2.34 + +-- !disabled_v2_true_parallel_false_native_and -- + +-- !disabled_v2_true_parallel_false_b_docvalue_control -- + +-- !disabled_v2_true_parallel_false_c_docvalue_control -- + +-- !disabled_v2_true_parallel_false_numeric_projection -- + +-- !disabled_v2_true_parallel_true_predicate_0 -- + +-- !disabled_v2_true_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_8 -- +2 empty \N \N + +-- !disabled_v2_true_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_true_predicate_13 -- + +-- !disabled_v2_true_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !disabled_v2_true_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !disabled_v2_true_parallel_true_filter_only -- + +-- !disabled_v2_true_parallel_true_native -- +ok 2.34 + +-- !disabled_v2_true_parallel_true_native_and -- + +-- !disabled_v2_true_parallel_true_b_docvalue_control -- + +-- !disabled_v2_true_parallel_true_c_docvalue_control -- + +-- !disabled_v2_true_parallel_true_numeric_projection -- + +-- !limited_v2_false_parallel_false_predicate_0 -- + +-- !limited_v2_false_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_8 -- +2 empty \N \N + +-- !limited_v2_false_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_false_predicate_13 -- + +-- !limited_v2_false_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_false_filter_only -- + +-- !limited_v2_false_parallel_false_native -- +ok 2.34 + +-- !limited_v2_false_parallel_false_native_and -- + +-- !limited_v2_false_parallel_false_b_docvalue_control -- +2.25 + +-- !limited_v2_false_parallel_false_c_docvalue_control -- +2.3398438 + +-- !limited_v2_false_parallel_false_numeric_projection -- + +-- !limited_v2_false_parallel_true_predicate_0 -- + +-- !limited_v2_false_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_8 -- +2 empty \N \N + +-- !limited_v2_false_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_true_predicate_13 -- + +-- !limited_v2_false_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !limited_v2_false_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_false_parallel_true_filter_only -- + +-- !limited_v2_false_parallel_true_native -- +ok 2.34 + +-- !limited_v2_false_parallel_true_native_and -- + +-- !limited_v2_false_parallel_true_b_docvalue_control -- +2.25 + +-- !limited_v2_false_parallel_true_c_docvalue_control -- +2.3398438 + +-- !limited_v2_false_parallel_true_numeric_projection -- + +-- !limited_v2_true_parallel_false_predicate_0 -- + +-- !limited_v2_true_parallel_false_predicate_1 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_false_predicate_2 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_false_predicate_3 -- +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_8 -- +2 empty \N \N + +-- !limited_v2_true_parallel_false_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_10 -- +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_predicate_12 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_false_predicate_13 -- + +-- !limited_v2_true_parallel_false_predicate_14 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_false_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_false_filter_only -- + +-- !limited_v2_true_parallel_false_native -- +ok 2.34 + +-- !limited_v2_true_parallel_false_native_and -- + +-- !limited_v2_true_parallel_false_b_docvalue_control -- +2.25 + +-- !limited_v2_true_parallel_false_c_docvalue_control -- +2.3398438 + +-- !limited_v2_true_parallel_false_numeric_projection -- + +-- !limited_v2_true_parallel_true_predicate_0 -- + +-- !limited_v2_true_parallel_true_predicate_1 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_true_predicate_2 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_true_predicate_3 -- +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_4 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_5 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_6 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_7 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_8 -- +2 empty \N \N + +-- !limited_v2_true_parallel_true_predicate_9 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_10 -- +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_11 -- +2 empty \N \N +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_predicate_12 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_true_predicate_13 -- + +-- !limited_v2_true_parallel_true_predicate_14 -- +1 ok 2.34 2.34 + +-- !limited_v2_true_parallel_true_predicate_15 -- +1 ok 2.34 2.34 +3 other -2.34 -2.34 + +-- !limited_v2_true_parallel_true_filter_only -- + +-- !limited_v2_true_parallel_true_native -- +ok 2.34 + +-- !limited_v2_true_parallel_true_native_and -- + +-- !limited_v2_true_parallel_true_b_docvalue_control -- +2.25 + +-- !limited_v2_true_parallel_true_c_docvalue_control -- +2.3398438 + +-- !limited_v2_true_parallel_true_numeric_projection -- + diff --git a/regression-test/data/external_table_p2/es/test_es_scalar_materialization.out b/regression-test/data/external_table_p2/es/test_es_scalar_materialization.out new file mode 100644 index 00000000000000..77098900808aba --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_scalar_materialization.out @@ -0,0 +1,989 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_true_v2_false_parallel_false_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_false_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_false_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_false_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_false_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_false_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_false_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_false_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_false_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_false_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_false_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_false_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_true_v2_false_parallel_false_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_true_v2_false_parallel_false_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_true_v2_false_parallel_false_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_true_v2_false_parallel_false_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_true_v2_false_parallel_false_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_true_v2_false_parallel_false_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_true_v2_false_parallel_false_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_true_v2_false_parallel_false_range_not_0 -- +2 2 +3 3 +5 1 + +-- !dv_true_v2_false_parallel_false_range_not_1 -- +3 3 +5 1 + +-- !dv_true_v2_false_parallel_true_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_true_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_true_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_true_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_false_parallel_true_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_true_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_true_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_true_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_false_parallel_true_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_true_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_true_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_false_parallel_true_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_true_v2_false_parallel_true_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_true_v2_false_parallel_true_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_true_v2_false_parallel_true_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_true_v2_false_parallel_true_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_true_v2_false_parallel_true_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_true_v2_false_parallel_true_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_true_v2_false_parallel_true_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_true_v2_false_parallel_true_range_not_0 -- +2 2 +3 3 +5 1 + +-- !dv_true_v2_false_parallel_true_range_not_1 -- +3 3 +5 1 + +-- !dv_true_v2_true_parallel_false_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_false_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_false_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_false_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_false_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_false_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_false_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_false_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_false_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_false_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_false_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_false_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_true_v2_true_parallel_false_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_true_v2_true_parallel_false_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_true_v2_true_parallel_false_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_true_v2_true_parallel_false_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_true_v2_true_parallel_false_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_true_v2_true_parallel_false_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_true_v2_true_parallel_false_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_true_v2_true_parallel_false_range_not_0 -- +2 2 +3 3 +5 1 + +-- !dv_true_v2_true_parallel_false_range_not_1 -- +3 3 +5 1 + +-- !dv_true_v2_true_parallel_true_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_true_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_true_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_true_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_true_v2_true_parallel_true_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_true_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_true_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_true_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_true_v2_true_parallel_true_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_true_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_true_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_true_v2_true_parallel_true_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_true_v2_true_parallel_true_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_true_v2_true_parallel_true_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_true_v2_true_parallel_true_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_true_v2_true_parallel_true_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_true_v2_true_parallel_true_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_true_v2_true_parallel_true_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_true_v2_true_parallel_true_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_true_v2_true_parallel_true_range_not_0 -- +2 2 +3 3 +5 1 + +-- !dv_true_v2_true_parallel_true_range_not_1 -- +3 3 +5 1 + +-- !dv_false_v2_false_parallel_false_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_false_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_false_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_false_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_false_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_false_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_false_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_false_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_false_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_false_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_false_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_false_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_false_v2_false_parallel_false_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_false_v2_false_parallel_false_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_false_v2_false_parallel_false_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_false_v2_false_parallel_false_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_false_v2_false_parallel_false_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_false_v2_false_parallel_false_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_false_v2_false_parallel_false_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_false_v2_false_parallel_true_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_true_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_true_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_true_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_false_parallel_true_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_true_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_true_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_true_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_false_parallel_true_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_true_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_true_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_false_parallel_true_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_false_v2_false_parallel_true_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_false_v2_false_parallel_true_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_false_v2_false_parallel_true_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_false_v2_false_parallel_true_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_false_v2_false_parallel_true_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_false_v2_false_parallel_true_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_false_v2_false_parallel_true_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_false_v2_true_parallel_false_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_false_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_false_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_false_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_false_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_false_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_false_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_false_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_false_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_false_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_false_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_false_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_false_v2_true_parallel_false_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_false_v2_true_parallel_false_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_false_v2_true_parallel_false_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_false_v2_true_parallel_false_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_false_v2_true_parallel_false_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_false_v2_true_parallel_false_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_false_v2_true_parallel_false_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + +-- !dv_false_v2_true_parallel_true_source_0 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_true_source_1 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_true_source_2 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_true_source_3 -- +2 two \N \N \N \N -1 -1 -1 0 [] +3 nullish \N \N \N \N \N \N \N \N \N +4 missing \N \N \N \N \N \N \N \N \N +7 nan \N \N \N \N 0 0 0 \N [0] + +-- !dv_false_v2_true_parallel_true_source_4 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +5 scalar 3.5 4.5 true 2024-01-01T00:00 127 32767 2147483647 9223372036854775807 [127] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_true_source_5 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_true_source_6 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_true_source_7 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] + +-- !dv_false_v2_true_parallel_true_source_8 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_true_source_9 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_true_source_10 -- +2 two \N \N \N \N -1 -1 -1 0 [] +6 negative 1.25 2.25 true 2024-01-01T08:00 -128 -32768 -2147483648 -9223372036854775808 [-128] + +-- !dv_false_v2_true_parallel_true_source_11 -- +1 one 1.5 2.5 false 2024-01-01T00:00 1 1 1 0 [1, -1] +2 two \N \N \N \N -1 -1 -1 0 [] + +-- !dv_false_v2_true_parallel_true_f_control -- +1 1.5 +2 \N +3 \N +4 \N +5 3.5 +6 1.25 +7 \N + +-- !dv_false_v2_true_parallel_true_d_control -- +1 2.5 +2 \N +3 \N +4 \N +5 4.5 +6 2.25 +7 \N + +-- !dv_false_v2_true_parallel_true_truth_control -- +1 false +2 \N +3 \N +4 \N +5 true +6 true +7 \N + +-- !dv_false_v2_true_parallel_true_n_control -- +1 1 +2 -1 +3 \N +4 \N +5 127 +6 -128 +7 0 + +-- !dv_false_v2_true_parallel_true_s_control -- +1 1 +2 -1 +3 \N +4 \N +5 32767 +6 -32768 +7 0 + +-- !dv_false_v2_true_parallel_true_i_control -- +1 1 +2 -1 +3 \N +4 \N +5 2147483647 +6 -2147483648 +7 0 + +-- !dv_false_v2_true_parallel_true_l_control -- +1 0 +2 0 +3 \N +4 \N +5 9223372036854775807 +6 -9223372036854775808 +7 \N + diff --git a/regression-test/data/external_table_p2/es/test_es_source_fallback_types.out b/regression-test/data/external_table_p2/es/test_es_source_fallback_types.out new file mode 100644 index 00000000000000..166e593307cc82 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_source_fallback_types.out @@ -0,0 +1,1141 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_true_v2_false_parallel_false_source_0 -- + +-- !dv_true_v2_false_parallel_false_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_false_source_2 -- + +-- !dv_true_v2_false_parallel_false_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_false_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_false_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_true_v2_false_parallel_false_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_true_v2_false_parallel_false_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_true_v2_false_parallel_false_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_true_v2_false_parallel_false_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_true_v2_false_parallel_false_alias_value_control -- +1 5 + +-- !dv_true_v2_false_parallel_false_generated_value_control -- +1 5 +2 5 +3 5 +4 5 + +-- !dv_true_v2_false_parallel_false_date_control -- +1 2024-01-01T08:00 +2 2024-01-01T08:00 + +-- !dv_true_v2_false_parallel_false_boolean_control -- +false + +-- !dv_true_v2_false_parallel_false_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_true_v2_false_parallel_false_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_true_v2_false_parallel_false_integer_1 -- +1 exponent 1200 + +-- !dv_true_v2_false_parallel_false_integer_control_1 -- +1 1200 + +-- !dv_true_v2_false_parallel_false_integer_2 -- +2 singleton 5 + +-- !dv_true_v2_false_parallel_false_integer_control_2 -- +2 5 + +-- !dv_true_v2_false_parallel_false_integer_3 -- +3 negative -19 + +-- !dv_true_v2_false_parallel_false_integer_control_3 -- +3 -19 + +-- !dv_true_v2_false_parallel_false_retained_not_0 -- +2 1 +3 2 + +-- !dv_true_v2_false_parallel_false_retained_not_1 -- +3 2 +4 2 + +-- !dv_true_v2_false_parallel_false_retained_not_2 -- +3 2 +4 2 + +-- !dv_true_v2_false_parallel_false_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_true_v2_false_parallel_false_root_0_0 -- + +-- !dv_true_v2_false_parallel_false_root_0_1 -- +ok 1 + +-- !dv_true_v2_false_parallel_false_root_1_0 -- +nullish \N + +-- !dv_true_v2_false_parallel_false_root_1_1 -- +ok 1 + +-- !dv_true_v2_false_parallel_false_disabled_flag_0 -- + +-- !dv_true_v2_false_parallel_false_disabled_rows_flag_0 -- +\N \N + +-- !dv_true_v2_false_parallel_false_disabled_flag_1 -- + +-- !dv_true_v2_false_parallel_false_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_false_parallel_false_disabled_mode_0 -- + +-- !dv_true_v2_false_parallel_false_disabled_rows_mode_0 -- +\N \N + +-- !dv_true_v2_false_parallel_false_disabled_mode_1 -- + +-- !dv_true_v2_false_parallel_false_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_false_parallel_true_source_0 -- + +-- !dv_true_v2_false_parallel_true_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_true_source_2 -- + +-- !dv_true_v2_false_parallel_true_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_true_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_true_v2_false_parallel_true_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_true_v2_false_parallel_true_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_true_v2_false_parallel_true_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_true_v2_false_parallel_true_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_true_v2_false_parallel_true_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_true_v2_false_parallel_true_alias_value_control -- +1 5 + +-- !dv_true_v2_false_parallel_true_generated_value_control -- +1 5 +2 5 +3 5 +4 5 + +-- !dv_true_v2_false_parallel_true_date_control -- +1 2024-01-01T08:00 +2 2024-01-01T08:00 + +-- !dv_true_v2_false_parallel_true_boolean_control -- +false + +-- !dv_true_v2_false_parallel_true_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_true_v2_false_parallel_true_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_true_v2_false_parallel_true_integer_1 -- +1 exponent 1200 + +-- !dv_true_v2_false_parallel_true_integer_control_1 -- +1 1200 + +-- !dv_true_v2_false_parallel_true_integer_2 -- +2 singleton 5 + +-- !dv_true_v2_false_parallel_true_integer_control_2 -- +2 5 + +-- !dv_true_v2_false_parallel_true_integer_3 -- +3 negative -19 + +-- !dv_true_v2_false_parallel_true_integer_control_3 -- +3 -19 + +-- !dv_true_v2_false_parallel_true_retained_not_0 -- +2 1 +3 2 + +-- !dv_true_v2_false_parallel_true_retained_not_1 -- +3 2 +4 2 + +-- !dv_true_v2_false_parallel_true_retained_not_2 -- +3 2 +4 2 + +-- !dv_true_v2_false_parallel_true_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_true_v2_false_parallel_true_root_0_0 -- + +-- !dv_true_v2_false_parallel_true_root_0_1 -- +ok 1 + +-- !dv_true_v2_false_parallel_true_root_1_0 -- +nullish \N + +-- !dv_true_v2_false_parallel_true_root_1_1 -- +ok 1 + +-- !dv_true_v2_false_parallel_true_disabled_flag_0 -- + +-- !dv_true_v2_false_parallel_true_disabled_rows_flag_0 -- +\N \N + +-- !dv_true_v2_false_parallel_true_disabled_flag_1 -- + +-- !dv_true_v2_false_parallel_true_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_false_parallel_true_disabled_mode_0 -- + +-- !dv_true_v2_false_parallel_true_disabled_rows_mode_0 -- +\N \N + +-- !dv_true_v2_false_parallel_true_disabled_mode_1 -- + +-- !dv_true_v2_false_parallel_true_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_true_parallel_false_source_0 -- + +-- !dv_true_v2_true_parallel_false_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_false_source_2 -- + +-- !dv_true_v2_true_parallel_false_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_false_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_false_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_true_v2_true_parallel_false_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_true_v2_true_parallel_false_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_true_v2_true_parallel_false_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_true_v2_true_parallel_false_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_true_v2_true_parallel_false_alias_value_control -- +1 5 + +-- !dv_true_v2_true_parallel_false_generated_value_control -- +1 5 +2 5 +3 5 +4 5 + +-- !dv_true_v2_true_parallel_false_date_control -- +1 2024-01-01T08:00 +2 2024-01-01T08:00 + +-- !dv_true_v2_true_parallel_false_boolean_control -- +false + +-- !dv_true_v2_true_parallel_false_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_true_v2_true_parallel_false_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_true_v2_true_parallel_false_integer_1 -- +1 exponent 1200 + +-- !dv_true_v2_true_parallel_false_integer_control_1 -- +1 1200 + +-- !dv_true_v2_true_parallel_false_integer_2 -- +2 singleton 5 + +-- !dv_true_v2_true_parallel_false_integer_control_2 -- +2 5 + +-- !dv_true_v2_true_parallel_false_integer_3 -- +3 negative -19 + +-- !dv_true_v2_true_parallel_false_integer_control_3 -- +3 -19 + +-- !dv_true_v2_true_parallel_false_retained_not_0 -- +2 1 +3 2 + +-- !dv_true_v2_true_parallel_false_retained_not_1 -- +3 2 +4 2 + +-- !dv_true_v2_true_parallel_false_retained_not_2 -- +3 2 +4 2 + +-- !dv_true_v2_true_parallel_false_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_true_v2_true_parallel_false_root_0_0 -- + +-- !dv_true_v2_true_parallel_false_root_0_1 -- +ok 1 + +-- !dv_true_v2_true_parallel_false_root_1_0 -- +nullish \N + +-- !dv_true_v2_true_parallel_false_root_1_1 -- +ok 1 + +-- !dv_true_v2_true_parallel_false_disabled_flag_0 -- + +-- !dv_true_v2_true_parallel_false_disabled_rows_flag_0 -- +\N \N + +-- !dv_true_v2_true_parallel_false_disabled_flag_1 -- + +-- !dv_true_v2_true_parallel_false_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_true_parallel_false_disabled_mode_0 -- + +-- !dv_true_v2_true_parallel_false_disabled_rows_mode_0 -- +\N \N + +-- !dv_true_v2_true_parallel_false_disabled_mode_1 -- + +-- !dv_true_v2_true_parallel_false_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_true_parallel_true_source_0 -- + +-- !dv_true_v2_true_parallel_true_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_true_source_2 -- + +-- !dv_true_v2_true_parallel_true_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_true_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_true_v2_true_parallel_true_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_true_v2_true_parallel_true_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_true_v2_true_parallel_true_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_true_v2_true_parallel_true_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_true_v2_true_parallel_true_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_true_v2_true_parallel_true_alias_value_control -- +1 5 + +-- !dv_true_v2_true_parallel_true_generated_value_control -- +1 5 +2 5 +3 5 +4 5 + +-- !dv_true_v2_true_parallel_true_date_control -- +1 2024-01-01T08:00 +2 2024-01-01T08:00 + +-- !dv_true_v2_true_parallel_true_boolean_control -- +false + +-- !dv_true_v2_true_parallel_true_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_true_v2_true_parallel_true_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_true_v2_true_parallel_true_integer_1 -- +1 exponent 1200 + +-- !dv_true_v2_true_parallel_true_integer_control_1 -- +1 1200 + +-- !dv_true_v2_true_parallel_true_integer_2 -- +2 singleton 5 + +-- !dv_true_v2_true_parallel_true_integer_control_2 -- +2 5 + +-- !dv_true_v2_true_parallel_true_integer_3 -- +3 negative -19 + +-- !dv_true_v2_true_parallel_true_integer_control_3 -- +3 -19 + +-- !dv_true_v2_true_parallel_true_retained_not_0 -- +2 1 +3 2 + +-- !dv_true_v2_true_parallel_true_retained_not_1 -- +3 2 +4 2 + +-- !dv_true_v2_true_parallel_true_retained_not_2 -- +3 2 +4 2 + +-- !dv_true_v2_true_parallel_true_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_true_v2_true_parallel_true_root_0_0 -- + +-- !dv_true_v2_true_parallel_true_root_0_1 -- +ok 1 + +-- !dv_true_v2_true_parallel_true_root_1_0 -- +nullish \N + +-- !dv_true_v2_true_parallel_true_root_1_1 -- +ok 1 + +-- !dv_true_v2_true_parallel_true_disabled_flag_0 -- + +-- !dv_true_v2_true_parallel_true_disabled_rows_flag_0 -- +\N \N + +-- !dv_true_v2_true_parallel_true_disabled_flag_1 -- + +-- !dv_true_v2_true_parallel_true_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_true_v2_true_parallel_true_disabled_mode_0 -- + +-- !dv_true_v2_true_parallel_true_disabled_rows_mode_0 -- +\N \N + +-- !dv_true_v2_true_parallel_true_disabled_mode_1 -- + +-- !dv_true_v2_true_parallel_true_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_false_parallel_false_source_0 -- + +-- !dv_false_v2_false_parallel_false_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_false_source_2 -- + +-- !dv_false_v2_false_parallel_false_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_false_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_false_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_false_v2_false_parallel_false_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_false_v2_false_parallel_false_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_false_v2_false_parallel_false_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_false_v2_false_parallel_false_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_false_v2_false_parallel_false_alias_value_control -- + +-- !dv_false_v2_false_parallel_false_generated_value_control -- + +-- !dv_false_v2_false_parallel_false_date_control -- +1 2024-01-01T00:00 +2 2024-01-01T08:00 + +-- !dv_false_v2_false_parallel_false_boolean_control -- +false + +-- !dv_false_v2_false_parallel_false_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_false_v2_false_parallel_false_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_false_v2_false_parallel_false_integer_1 -- +1 exponent 1200 + +-- !dv_false_v2_false_parallel_false_integer_control_1 -- +1 1200 + +-- !dv_false_v2_false_parallel_false_integer_2 -- +2 singleton 5 + +-- !dv_false_v2_false_parallel_false_integer_control_2 -- +2 5 + +-- !dv_false_v2_false_parallel_false_integer_3 -- +3 negative -19 + +-- !dv_false_v2_false_parallel_false_integer_control_3 -- +3 -19 + +-- !dv_false_v2_false_parallel_false_retained_not_0 -- +2 1 +3 2 + +-- !dv_false_v2_false_parallel_false_retained_not_1 -- +3 2 +4 2 + +-- !dv_false_v2_false_parallel_false_retained_not_2 -- +3 2 +4 2 + +-- !dv_false_v2_false_parallel_false_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_false_v2_false_parallel_false_root_0_0 -- + +-- !dv_false_v2_false_parallel_false_root_0_1 -- +ok 1 + +-- !dv_false_v2_false_parallel_false_root_1_0 -- +nullish \N + +-- !dv_false_v2_false_parallel_false_root_1_1 -- +ok 1 + +-- !dv_false_v2_false_parallel_false_disabled_flag_0 -- + +-- !dv_false_v2_false_parallel_false_disabled_rows_flag_0 -- +\N \N + +-- !dv_false_v2_false_parallel_false_disabled_flag_1 -- + +-- !dv_false_v2_false_parallel_false_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_false_parallel_false_disabled_mode_0 -- + +-- !dv_false_v2_false_parallel_false_disabled_rows_mode_0 -- +\N \N + +-- !dv_false_v2_false_parallel_false_disabled_mode_1 -- + +-- !dv_false_v2_false_parallel_false_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_false_parallel_true_source_0 -- + +-- !dv_false_v2_false_parallel_true_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_true_source_2 -- + +-- !dv_false_v2_false_parallel_true_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_true_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_false_v2_false_parallel_true_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_false_v2_false_parallel_true_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_false_v2_false_parallel_true_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_false_v2_false_parallel_true_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_false_v2_false_parallel_true_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_false_v2_false_parallel_true_alias_value_control -- + +-- !dv_false_v2_false_parallel_true_generated_value_control -- + +-- !dv_false_v2_false_parallel_true_date_control -- +1 2024-01-01T00:00 +2 2024-01-01T08:00 + +-- !dv_false_v2_false_parallel_true_boolean_control -- +false + +-- !dv_false_v2_false_parallel_true_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_false_v2_false_parallel_true_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_false_v2_false_parallel_true_integer_1 -- +1 exponent 1200 + +-- !dv_false_v2_false_parallel_true_integer_control_1 -- +1 1200 + +-- !dv_false_v2_false_parallel_true_integer_2 -- +2 singleton 5 + +-- !dv_false_v2_false_parallel_true_integer_control_2 -- +2 5 + +-- !dv_false_v2_false_parallel_true_integer_3 -- +3 negative -19 + +-- !dv_false_v2_false_parallel_true_integer_control_3 -- +3 -19 + +-- !dv_false_v2_false_parallel_true_retained_not_0 -- +2 1 +3 2 + +-- !dv_false_v2_false_parallel_true_retained_not_1 -- +3 2 +4 2 + +-- !dv_false_v2_false_parallel_true_retained_not_2 -- +3 2 +4 2 + +-- !dv_false_v2_false_parallel_true_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_false_v2_false_parallel_true_root_0_0 -- + +-- !dv_false_v2_false_parallel_true_root_0_1 -- +ok 1 + +-- !dv_false_v2_false_parallel_true_root_1_0 -- +nullish \N + +-- !dv_false_v2_false_parallel_true_root_1_1 -- +ok 1 + +-- !dv_false_v2_false_parallel_true_disabled_flag_0 -- + +-- !dv_false_v2_false_parallel_true_disabled_rows_flag_0 -- +\N \N + +-- !dv_false_v2_false_parallel_true_disabled_flag_1 -- + +-- !dv_false_v2_false_parallel_true_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_false_parallel_true_disabled_mode_0 -- + +-- !dv_false_v2_false_parallel_true_disabled_rows_mode_0 -- +\N \N + +-- !dv_false_v2_false_parallel_true_disabled_mode_1 -- + +-- !dv_false_v2_false_parallel_true_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_true_parallel_false_source_0 -- + +-- !dv_false_v2_true_parallel_false_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_false_source_2 -- + +-- !dv_false_v2_true_parallel_false_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_false_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_false_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_false_v2_true_parallel_false_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_false_v2_true_parallel_false_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_false_v2_true_parallel_false_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_false_v2_true_parallel_false_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_false_v2_true_parallel_false_alias_value_control -- + +-- !dv_false_v2_true_parallel_false_generated_value_control -- + +-- !dv_false_v2_true_parallel_false_date_control -- +1 2024-01-01T00:00 +2 2024-01-01T08:00 + +-- !dv_false_v2_true_parallel_false_boolean_control -- +false + +-- !dv_false_v2_true_parallel_false_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_false_v2_true_parallel_false_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_false_v2_true_parallel_false_integer_1 -- +1 exponent 1200 + +-- !dv_false_v2_true_parallel_false_integer_control_1 -- +1 1200 + +-- !dv_false_v2_true_parallel_false_integer_2 -- +2 singleton 5 + +-- !dv_false_v2_true_parallel_false_integer_control_2 -- +2 5 + +-- !dv_false_v2_true_parallel_false_integer_3 -- +3 negative -19 + +-- !dv_false_v2_true_parallel_false_integer_control_3 -- +3 -19 + +-- !dv_false_v2_true_parallel_false_retained_not_0 -- +2 1 +3 2 + +-- !dv_false_v2_true_parallel_false_retained_not_1 -- +3 2 +4 2 + +-- !dv_false_v2_true_parallel_false_retained_not_2 -- +3 2 +4 2 + +-- !dv_false_v2_true_parallel_false_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_false_v2_true_parallel_false_root_0_0 -- + +-- !dv_false_v2_true_parallel_false_root_0_1 -- +ok 1 + +-- !dv_false_v2_true_parallel_false_root_1_0 -- +nullish \N + +-- !dv_false_v2_true_parallel_false_root_1_1 -- +ok 1 + +-- !dv_false_v2_true_parallel_false_disabled_flag_0 -- + +-- !dv_false_v2_true_parallel_false_disabled_rows_flag_0 -- +\N \N + +-- !dv_false_v2_true_parallel_false_disabled_flag_1 -- + +-- !dv_false_v2_true_parallel_false_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_true_parallel_false_disabled_mode_0 -- + +-- !dv_false_v2_true_parallel_false_disabled_rows_mode_0 -- +\N \N + +-- !dv_false_v2_true_parallel_false_disabled_mode_1 -- + +-- !dv_false_v2_true_parallel_false_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_true_parallel_true_source_0 -- + +-- !dv_false_v2_true_parallel_true_source_1 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_true_source_2 -- + +-- !dv_false_v2_true_parallel_true_source_3 -- +1 ok \N +2 other \N +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_true_source_4 -- +3 nullish \N +4 absent \N + +-- !dv_false_v2_true_parallel_true_source_5 -- +1 ok [] +2 other [1, 2] + +-- !dv_false_v2_true_parallel_true_source_6 -- +2 other 2024-01-01T08:00 + +-- !dv_false_v2_true_parallel_true_source_7 -- +1 ok 2024-01-01T00:00 + +-- !dv_false_v2_true_parallel_true_source_8 -- +1 ok false [0, 1, 0] +2 other false [1] +3 nullish false [] +4 absent true \N + +-- !dv_false_v2_true_parallel_true_source_9 -- +1 ok false +2 other false +3 nullish false + +-- !dv_false_v2_true_parallel_true_alias_value_control -- + +-- !dv_false_v2_true_parallel_true_generated_value_control -- + +-- !dv_false_v2_true_parallel_true_date_control -- +1 2024-01-01T00:00 +2 2024-01-01T08:00 + +-- !dv_false_v2_true_parallel_true_boolean_control -- +false + +-- !dv_false_v2_true_parallel_true_integer_0 -- +1 exponent 1200 +2 singleton 5 +3 negative -19 +4 maximum 9223372036854775807 +5 fraction 0 + +-- !dv_false_v2_true_parallel_true_integer_control_0 -- +1 1200 +2 5 +3 -19 +4 9223372036854775807 +5 0 + +-- !dv_false_v2_true_parallel_true_integer_1 -- +1 exponent 1200 + +-- !dv_false_v2_true_parallel_true_integer_control_1 -- +1 1200 + +-- !dv_false_v2_true_parallel_true_integer_2 -- +2 singleton 5 + +-- !dv_false_v2_true_parallel_true_integer_control_2 -- +2 5 + +-- !dv_false_v2_true_parallel_true_integer_3 -- +3 negative -19 + +-- !dv_false_v2_true_parallel_true_integer_control_3 -- +3 -19 + +-- !dv_false_v2_true_parallel_true_retained_not_0 -- +2 1 +3 2 + +-- !dv_false_v2_true_parallel_true_retained_not_1 -- +3 2 +4 2 + +-- !dv_false_v2_true_parallel_true_retained_not_2 -- +3 2 +4 2 + +-- !dv_false_v2_true_parallel_true_retained_not_3 -- +2 1 +3 2 +4 2 + +-- !dv_false_v2_true_parallel_true_root_0_0 -- + +-- !dv_false_v2_true_parallel_true_root_0_1 -- +ok 1 + +-- !dv_false_v2_true_parallel_true_root_1_0 -- +nullish \N + +-- !dv_false_v2_true_parallel_true_root_1_1 -- +ok 1 + +-- !dv_false_v2_true_parallel_true_disabled_flag_0 -- + +-- !dv_false_v2_true_parallel_true_disabled_rows_flag_0 -- +\N \N + +-- !dv_false_v2_true_parallel_true_disabled_flag_1 -- + +-- !dv_false_v2_true_parallel_true_disabled_rows_flag_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + +-- !dv_false_v2_true_parallel_true_disabled_mode_0 -- + +-- !dv_false_v2_true_parallel_true_disabled_rows_mode_0 -- +\N \N + +-- !dv_false_v2_true_parallel_true_disabled_mode_1 -- + +-- !dv_false_v2_true_parallel_true_disabled_rows_mode_1 -- +\N \N +absent 0 +nullish \N +ok 5 +other 7 + diff --git a/regression-test/data/external_table_p2/es/test_es_source_semantics.out b/regression-test/data/external_table_p2/es/test_es_source_semantics.out new file mode 100644 index 00000000000000..9cf93ecdadb022 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_source_semantics.out @@ -0,0 +1,2509 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_false_v2_false_parallel_false_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_false_v2_false_parallel_false_predicate_0 -- +1 + +-- !dv_false_v2_false_parallel_false_predicate_1 -- +1 +2 + +-- !dv_false_v2_false_parallel_false_predicate_2 -- +1 +3 + +-- !dv_false_v2_false_parallel_false_predicate_3 -- +4 + +-- !dv_false_v2_false_parallel_false_predicate_4 -- +1 +2 +3 + +-- !dv_false_v2_false_parallel_false_predicate_5 -- +4 + +-- !dv_false_v2_false_parallel_false_predicate_6 -- +1 + +-- !dv_false_v2_false_parallel_false_predicate_7 -- +1 + +-- !dv_false_v2_false_parallel_false_predicate_8 -- +1 + +-- !dv_false_v2_false_parallel_false_predicate_9 -- +1 +2 + +-- !dv_false_v2_false_parallel_false_predicate_10 -- +1 + +-- !dv_false_v2_false_parallel_false_predicate_11 -- + +-- !dv_false_v2_false_parallel_false_predicate_12 -- +1 +3 + +-- !dv_false_v2_false_parallel_false_predicate_13 -- +1 +2 + +-- !dv_false_v2_false_parallel_false_predicate_14 -- +3 +4 + +-- !dv_false_v2_false_parallel_false_predicate_15 -- +3 + +-- !dv_false_v2_false_parallel_false_predicate_16 -- +1 +2 +4 + +-- !dv_false_v2_false_parallel_false_predicate_17 -- + +-- !dv_false_v2_false_parallel_false_predicate_18 -- +1 +2 + +-- !dv_false_v2_false_parallel_false_predicate_19 -- +2 +3 +4 + +-- !dv_false_v2_false_parallel_false_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_false_v2_false_parallel_false_native_3 -- + +-- !dv_false_v2_false_parallel_false_native_4 -- + +-- !dv_false_v2_false_parallel_false_native_5 -- +1 + +-- !dv_false_v2_false_parallel_false_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_false_v2_false_parallel_false_array_predicate_0 -- +5 + +-- !dv_false_v2_false_parallel_false_array_predicate_1 -- +5 + +-- !dv_false_v2_false_parallel_false_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_false_array_predicate_3 -- +1 + +-- !dv_false_v2_false_parallel_false_array_predicate_4 -- + +-- !dv_false_v2_false_parallel_false_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_false_v2_false_parallel_false_array_predicate_6 -- +5 + +-- !dv_false_v2_false_parallel_false_array_predicate_7 -- +5 + +-- !dv_false_v2_false_parallel_false_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_false_array_predicate_9 -- + +-- !dv_false_v2_false_parallel_false_array_predicate_10 -- +5 +6 + +-- !dv_false_v2_false_parallel_false_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_false_sibling_0 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_sibling_1 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_false_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_false_v2_false_parallel_false_sibling_4 -- +6 elephant \N + +-- !dv_false_v2_false_parallel_false_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_false_sibling_6 -- +6 elephant \N + +-- !dv_false_v2_false_parallel_false_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_false_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_false_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_false_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_sibling_11 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_false_v2_false_parallel_false_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_false_v2_false_parallel_false_sibling_limit -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_sibling_docvalue_control -- + +-- !dv_false_v2_false_parallel_false_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_false_v2_false_parallel_false_sibling_native_and -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_false_native_arrays -- +1 +2 +5 + +-- !dv_false_v2_false_parallel_false_limit_0 -- +1 + +-- !dv_false_v2_false_parallel_false_limit_1 -- +4 + +-- !dv_false_v2_false_parallel_false_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_false_v2_false_parallel_false_doris_source_filtered -- +1 +3 + +-- !dv_false_v2_false_parallel_false_doris_source_filtered_count -- +2 + +-- !dv_false_v2_false_parallel_false_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_false_v2_false_parallel_false_doris_source_multi_count -- +4 + +-- !dv_false_v2_false_parallel_false_doris_source_routed -- +1 +2 + +-- !dv_false_v2_false_parallel_false_doris_source_routed_count -- +2 + +-- !dv_false_v2_false_parallel_false_doris_source_filtered_routed -- +1 + +-- !dv_false_v2_false_parallel_false_doris_source_filtered_routed_count -- +1 + +-- !dv_false_v2_false_parallel_false_doris_source_empty -- + +-- !dv_false_v2_false_parallel_false_doris_source_empty_count -- +0 + +-- !dv_false_v2_false_parallel_false_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_false_v2_false_parallel_false_doris_source_unfiltered_count -- +8 + +-- !dv_false_v2_false_parallel_false_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_false_v2_false_parallel_false_doris_source_with_empty_count -- +4 + +-- !dv_false_v2_false_parallel_true_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_false_v2_false_parallel_true_predicate_0 -- +1 + +-- !dv_false_v2_false_parallel_true_predicate_1 -- +1 +2 + +-- !dv_false_v2_false_parallel_true_predicate_2 -- +1 +3 + +-- !dv_false_v2_false_parallel_true_predicate_3 -- +4 + +-- !dv_false_v2_false_parallel_true_predicate_4 -- +1 +2 +3 + +-- !dv_false_v2_false_parallel_true_predicate_5 -- +4 + +-- !dv_false_v2_false_parallel_true_predicate_6 -- +1 + +-- !dv_false_v2_false_parallel_true_predicate_7 -- +1 + +-- !dv_false_v2_false_parallel_true_predicate_8 -- +1 + +-- !dv_false_v2_false_parallel_true_predicate_9 -- +1 +2 + +-- !dv_false_v2_false_parallel_true_predicate_10 -- +1 + +-- !dv_false_v2_false_parallel_true_predicate_11 -- + +-- !dv_false_v2_false_parallel_true_predicate_12 -- +1 +3 + +-- !dv_false_v2_false_parallel_true_predicate_13 -- +1 +2 + +-- !dv_false_v2_false_parallel_true_predicate_14 -- +3 +4 + +-- !dv_false_v2_false_parallel_true_predicate_15 -- +3 + +-- !dv_false_v2_false_parallel_true_predicate_16 -- +1 +2 +4 + +-- !dv_false_v2_false_parallel_true_predicate_17 -- + +-- !dv_false_v2_false_parallel_true_predicate_18 -- +1 +2 + +-- !dv_false_v2_false_parallel_true_predicate_19 -- +2 +3 +4 + +-- !dv_false_v2_false_parallel_true_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_false_v2_false_parallel_true_native_3 -- + +-- !dv_false_v2_false_parallel_true_native_4 -- + +-- !dv_false_v2_false_parallel_true_native_5 -- +1 + +-- !dv_false_v2_false_parallel_true_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_false_v2_false_parallel_true_array_predicate_0 -- +5 + +-- !dv_false_v2_false_parallel_true_array_predicate_1 -- +5 + +-- !dv_false_v2_false_parallel_true_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_true_array_predicate_3 -- +1 + +-- !dv_false_v2_false_parallel_true_array_predicate_4 -- + +-- !dv_false_v2_false_parallel_true_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_false_v2_false_parallel_true_array_predicate_6 -- +5 + +-- !dv_false_v2_false_parallel_true_array_predicate_7 -- +5 + +-- !dv_false_v2_false_parallel_true_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_true_array_predicate_9 -- + +-- !dv_false_v2_false_parallel_true_array_predicate_10 -- +5 +6 + +-- !dv_false_v2_false_parallel_true_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_false_parallel_true_sibling_0 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_sibling_1 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_true_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_false_v2_false_parallel_true_sibling_4 -- +6 elephant \N + +-- !dv_false_v2_false_parallel_true_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_true_sibling_6 -- +6 elephant \N + +-- !dv_false_v2_false_parallel_true_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_true_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_true_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_false_parallel_true_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_sibling_11 -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_false_v2_false_parallel_true_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_false_v2_false_parallel_true_sibling_limit -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_sibling_docvalue_control -- + +-- !dv_false_v2_false_parallel_true_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_false_v2_false_parallel_true_sibling_native_and -- +2 ["cat"] cat + +-- !dv_false_v2_false_parallel_true_native_arrays -- +1 +2 +5 + +-- !dv_false_v2_false_parallel_true_limit_0 -- +1 + +-- !dv_false_v2_false_parallel_true_limit_1 -- +4 + +-- !dv_false_v2_false_parallel_true_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_false_v2_false_parallel_true_doris_source_filtered -- +1 +3 + +-- !dv_false_v2_false_parallel_true_doris_source_filtered_count -- +2 + +-- !dv_false_v2_false_parallel_true_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_false_v2_false_parallel_true_doris_source_multi_count -- +4 + +-- !dv_false_v2_false_parallel_true_doris_source_routed -- +1 +2 + +-- !dv_false_v2_false_parallel_true_doris_source_routed_count -- +2 + +-- !dv_false_v2_false_parallel_true_doris_source_filtered_routed -- +1 + +-- !dv_false_v2_false_parallel_true_doris_source_filtered_routed_count -- +1 + +-- !dv_false_v2_false_parallel_true_doris_source_empty -- + +-- !dv_false_v2_false_parallel_true_doris_source_empty_count -- +0 + +-- !dv_false_v2_false_parallel_true_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_false_v2_false_parallel_true_doris_source_unfiltered_count -- +8 + +-- !dv_false_v2_false_parallel_true_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_false_v2_false_parallel_true_doris_source_with_empty_count -- +4 + +-- !dv_false_v2_true_parallel_false_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_false_v2_true_parallel_false_predicate_0 -- +1 + +-- !dv_false_v2_true_parallel_false_predicate_1 -- +1 +2 + +-- !dv_false_v2_true_parallel_false_predicate_2 -- +1 +3 + +-- !dv_false_v2_true_parallel_false_predicate_3 -- +4 + +-- !dv_false_v2_true_parallel_false_predicate_4 -- +1 +2 +3 + +-- !dv_false_v2_true_parallel_false_predicate_5 -- +4 + +-- !dv_false_v2_true_parallel_false_predicate_6 -- +1 + +-- !dv_false_v2_true_parallel_false_predicate_7 -- +1 + +-- !dv_false_v2_true_parallel_false_predicate_8 -- +1 + +-- !dv_false_v2_true_parallel_false_predicate_9 -- +1 +2 + +-- !dv_false_v2_true_parallel_false_predicate_10 -- +1 + +-- !dv_false_v2_true_parallel_false_predicate_11 -- + +-- !dv_false_v2_true_parallel_false_predicate_12 -- +1 +3 + +-- !dv_false_v2_true_parallel_false_predicate_13 -- +1 +2 + +-- !dv_false_v2_true_parallel_false_predicate_14 -- +3 +4 + +-- !dv_false_v2_true_parallel_false_predicate_15 -- +3 + +-- !dv_false_v2_true_parallel_false_predicate_16 -- +1 +2 +4 + +-- !dv_false_v2_true_parallel_false_predicate_17 -- + +-- !dv_false_v2_true_parallel_false_predicate_18 -- +1 +2 + +-- !dv_false_v2_true_parallel_false_predicate_19 -- +2 +3 +4 + +-- !dv_false_v2_true_parallel_false_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_false_v2_true_parallel_false_native_3 -- + +-- !dv_false_v2_true_parallel_false_native_4 -- + +-- !dv_false_v2_true_parallel_false_native_5 -- +1 + +-- !dv_false_v2_true_parallel_false_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_false_v2_true_parallel_false_array_predicate_0 -- +5 + +-- !dv_false_v2_true_parallel_false_array_predicate_1 -- +5 + +-- !dv_false_v2_true_parallel_false_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_false_array_predicate_3 -- +1 + +-- !dv_false_v2_true_parallel_false_array_predicate_4 -- + +-- !dv_false_v2_true_parallel_false_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_false_v2_true_parallel_false_array_predicate_6 -- +5 + +-- !dv_false_v2_true_parallel_false_array_predicate_7 -- +5 + +-- !dv_false_v2_true_parallel_false_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_false_array_predicate_9 -- + +-- !dv_false_v2_true_parallel_false_array_predicate_10 -- +5 +6 + +-- !dv_false_v2_true_parallel_false_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_false_sibling_0 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_sibling_1 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_false_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_false_v2_true_parallel_false_sibling_4 -- +6 elephant \N + +-- !dv_false_v2_true_parallel_false_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_false_sibling_6 -- +6 elephant \N + +-- !dv_false_v2_true_parallel_false_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_false_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_false_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_false_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_sibling_11 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_false_v2_true_parallel_false_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_false_v2_true_parallel_false_sibling_limit -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_sibling_docvalue_control -- + +-- !dv_false_v2_true_parallel_false_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_false_v2_true_parallel_false_sibling_native_and -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_false_native_arrays -- +1 +2 +5 + +-- !dv_false_v2_true_parallel_false_limit_0 -- +1 + +-- !dv_false_v2_true_parallel_false_limit_1 -- +4 + +-- !dv_false_v2_true_parallel_false_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_false_v2_true_parallel_false_doris_source_filtered -- +1 +3 + +-- !dv_false_v2_true_parallel_false_doris_source_filtered_count -- +2 + +-- !dv_false_v2_true_parallel_false_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_false_v2_true_parallel_false_doris_source_multi_count -- +4 + +-- !dv_false_v2_true_parallel_false_doris_source_routed -- +1 +2 + +-- !dv_false_v2_true_parallel_false_doris_source_routed_count -- +2 + +-- !dv_false_v2_true_parallel_false_doris_source_filtered_routed -- +1 + +-- !dv_false_v2_true_parallel_false_doris_source_filtered_routed_count -- +1 + +-- !dv_false_v2_true_parallel_false_doris_source_empty -- + +-- !dv_false_v2_true_parallel_false_doris_source_empty_count -- +0 + +-- !dv_false_v2_true_parallel_false_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_false_v2_true_parallel_false_doris_source_unfiltered_count -- +8 + +-- !dv_false_v2_true_parallel_false_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_false_v2_true_parallel_false_doris_source_with_empty_count -- +4 + +-- !dv_false_v2_true_parallel_true_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_false_v2_true_parallel_true_predicate_0 -- +1 + +-- !dv_false_v2_true_parallel_true_predicate_1 -- +1 +2 + +-- !dv_false_v2_true_parallel_true_predicate_2 -- +1 +3 + +-- !dv_false_v2_true_parallel_true_predicate_3 -- +4 + +-- !dv_false_v2_true_parallel_true_predicate_4 -- +1 +2 +3 + +-- !dv_false_v2_true_parallel_true_predicate_5 -- +4 + +-- !dv_false_v2_true_parallel_true_predicate_6 -- +1 + +-- !dv_false_v2_true_parallel_true_predicate_7 -- +1 + +-- !dv_false_v2_true_parallel_true_predicate_8 -- +1 + +-- !dv_false_v2_true_parallel_true_predicate_9 -- +1 +2 + +-- !dv_false_v2_true_parallel_true_predicate_10 -- +1 + +-- !dv_false_v2_true_parallel_true_predicate_11 -- + +-- !dv_false_v2_true_parallel_true_predicate_12 -- +1 +3 + +-- !dv_false_v2_true_parallel_true_predicate_13 -- +1 +2 + +-- !dv_false_v2_true_parallel_true_predicate_14 -- +3 +4 + +-- !dv_false_v2_true_parallel_true_predicate_15 -- +3 + +-- !dv_false_v2_true_parallel_true_predicate_16 -- +1 +2 +4 + +-- !dv_false_v2_true_parallel_true_predicate_17 -- + +-- !dv_false_v2_true_parallel_true_predicate_18 -- +1 +2 + +-- !dv_false_v2_true_parallel_true_predicate_19 -- +2 +3 +4 + +-- !dv_false_v2_true_parallel_true_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_false_v2_true_parallel_true_native_3 -- + +-- !dv_false_v2_true_parallel_true_native_4 -- + +-- !dv_false_v2_true_parallel_true_native_5 -- +1 + +-- !dv_false_v2_true_parallel_true_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_false_v2_true_parallel_true_array_predicate_0 -- +5 + +-- !dv_false_v2_true_parallel_true_array_predicate_1 -- +5 + +-- !dv_false_v2_true_parallel_true_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_true_array_predicate_3 -- +1 + +-- !dv_false_v2_true_parallel_true_array_predicate_4 -- + +-- !dv_false_v2_true_parallel_true_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_false_v2_true_parallel_true_array_predicate_6 -- +5 + +-- !dv_false_v2_true_parallel_true_array_predicate_7 -- +5 + +-- !dv_false_v2_true_parallel_true_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_true_array_predicate_9 -- + +-- !dv_false_v2_true_parallel_true_array_predicate_10 -- +5 +6 + +-- !dv_false_v2_true_parallel_true_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_false_v2_true_parallel_true_sibling_0 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_sibling_1 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_true_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_false_v2_true_parallel_true_sibling_4 -- +6 elephant \N + +-- !dv_false_v2_true_parallel_true_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_true_sibling_6 -- +6 elephant \N + +-- !dv_false_v2_true_parallel_true_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_true_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_true_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_false_v2_true_parallel_true_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_sibling_11 -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_false_v2_true_parallel_true_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_false_v2_true_parallel_true_sibling_limit -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_sibling_docvalue_control -- + +-- !dv_false_v2_true_parallel_true_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_false_v2_true_parallel_true_sibling_native_and -- +2 ["cat"] cat + +-- !dv_false_v2_true_parallel_true_native_arrays -- +1 +2 +5 + +-- !dv_false_v2_true_parallel_true_limit_0 -- +1 + +-- !dv_false_v2_true_parallel_true_limit_1 -- +4 + +-- !dv_false_v2_true_parallel_true_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_false_v2_true_parallel_true_doris_source_filtered -- +1 +3 + +-- !dv_false_v2_true_parallel_true_doris_source_filtered_count -- +2 + +-- !dv_false_v2_true_parallel_true_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_false_v2_true_parallel_true_doris_source_multi_count -- +4 + +-- !dv_false_v2_true_parallel_true_doris_source_routed -- +1 +2 + +-- !dv_false_v2_true_parallel_true_doris_source_routed_count -- +2 + +-- !dv_false_v2_true_parallel_true_doris_source_filtered_routed -- +1 + +-- !dv_false_v2_true_parallel_true_doris_source_filtered_routed_count -- +1 + +-- !dv_false_v2_true_parallel_true_doris_source_empty -- + +-- !dv_false_v2_true_parallel_true_doris_source_empty_count -- +0 + +-- !dv_false_v2_true_parallel_true_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_false_v2_true_parallel_true_doris_source_unfiltered_count -- +8 + +-- !dv_false_v2_true_parallel_true_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_false_v2_true_parallel_true_doris_source_with_empty_count -- +4 + +-- !dv_true_v2_false_parallel_false_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_true_v2_false_parallel_false_predicate_0 -- +1 + +-- !dv_true_v2_false_parallel_false_predicate_1 -- +1 +2 + +-- !dv_true_v2_false_parallel_false_predicate_2 -- +1 +3 + +-- !dv_true_v2_false_parallel_false_predicate_3 -- +4 + +-- !dv_true_v2_false_parallel_false_predicate_4 -- +1 +2 +3 + +-- !dv_true_v2_false_parallel_false_predicate_5 -- +4 + +-- !dv_true_v2_false_parallel_false_predicate_6 -- +1 + +-- !dv_true_v2_false_parallel_false_predicate_7 -- +1 + +-- !dv_true_v2_false_parallel_false_predicate_8 -- +1 + +-- !dv_true_v2_false_parallel_false_predicate_9 -- +1 +2 + +-- !dv_true_v2_false_parallel_false_predicate_10 -- +1 + +-- !dv_true_v2_false_parallel_false_predicate_11 -- + +-- !dv_true_v2_false_parallel_false_predicate_12 -- +1 +3 + +-- !dv_true_v2_false_parallel_false_predicate_13 -- +1 +2 + +-- !dv_true_v2_false_parallel_false_predicate_14 -- +3 +4 + +-- !dv_true_v2_false_parallel_false_predicate_15 -- +3 + +-- !dv_true_v2_false_parallel_false_predicate_16 -- +1 +2 +4 + +-- !dv_true_v2_false_parallel_false_predicate_17 -- + +-- !dv_true_v2_false_parallel_false_predicate_18 -- +1 +2 + +-- !dv_true_v2_false_parallel_false_predicate_19 -- +2 +3 +4 + +-- !dv_true_v2_false_parallel_false_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_true_v2_false_parallel_false_native_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_false_native_1 -- +1 +3 + +-- !dv_true_v2_false_parallel_false_native_2 -- +1 +3 + +-- !dv_true_v2_false_parallel_false_native_3 -- + +-- !dv_true_v2_false_parallel_false_native_4 -- + +-- !dv_true_v2_false_parallel_false_native_5 -- +1 + +-- !dv_true_v2_false_parallel_false_preserved_source_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_false_preserved_source_1 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_false_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_true_v2_false_parallel_false_array_predicate_0 -- +5 + +-- !dv_true_v2_false_parallel_false_array_predicate_1 -- +5 + +-- !dv_true_v2_false_parallel_false_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_false_array_predicate_3 -- +1 + +-- !dv_true_v2_false_parallel_false_array_predicate_4 -- + +-- !dv_true_v2_false_parallel_false_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_true_v2_false_parallel_false_array_predicate_6 -- +5 + +-- !dv_true_v2_false_parallel_false_array_predicate_7 -- +5 + +-- !dv_true_v2_false_parallel_false_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_false_array_predicate_9 -- + +-- !dv_true_v2_false_parallel_false_array_predicate_10 -- +5 +6 + +-- !dv_true_v2_false_parallel_false_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_false_sibling_0 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_sibling_1 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_false_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_true_v2_false_parallel_false_sibling_4 -- +6 elephant \N + +-- !dv_true_v2_false_parallel_false_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_false_sibling_6 -- +6 elephant \N + +-- !dv_true_v2_false_parallel_false_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_false_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_false_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_false_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_sibling_11 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_true_v2_false_parallel_false_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_true_v2_false_parallel_false_sibling_limit -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_sibling_docvalue_control -- +cat + +-- !dv_true_v2_false_parallel_false_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_true_v2_false_parallel_false_sibling_native_and -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_false_native_arrays -- +1 +2 +5 + +-- !dv_true_v2_false_parallel_false_limit_0 -- +1 + +-- !dv_true_v2_false_parallel_false_limit_1 -- +4 + +-- !dv_true_v2_false_parallel_false_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_true_v2_false_parallel_false_doris_source_filtered -- +1 +3 + +-- !dv_true_v2_false_parallel_false_doris_source_filtered_count -- +2 + +-- !dv_true_v2_false_parallel_false_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_true_v2_false_parallel_false_doris_source_multi_count -- +4 + +-- !dv_true_v2_false_parallel_false_doris_source_routed -- +1 +2 + +-- !dv_true_v2_false_parallel_false_doris_source_routed_count -- +2 + +-- !dv_true_v2_false_parallel_false_doris_source_filtered_routed -- +1 + +-- !dv_true_v2_false_parallel_false_doris_source_filtered_routed_count -- +1 + +-- !dv_true_v2_false_parallel_false_doris_source_empty -- + +-- !dv_true_v2_false_parallel_false_doris_source_empty_count -- +0 + +-- !dv_true_v2_false_parallel_false_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_true_v2_false_parallel_false_doris_source_unfiltered_count -- +8 + +-- !dv_true_v2_false_parallel_false_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_false_doris_source_with_empty_count -- +4 + +-- !dv_true_v2_false_parallel_true_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_true_v2_false_parallel_true_predicate_0 -- +1 + +-- !dv_true_v2_false_parallel_true_predicate_1 -- +1 +2 + +-- !dv_true_v2_false_parallel_true_predicate_2 -- +1 +3 + +-- !dv_true_v2_false_parallel_true_predicate_3 -- +4 + +-- !dv_true_v2_false_parallel_true_predicate_4 -- +1 +2 +3 + +-- !dv_true_v2_false_parallel_true_predicate_5 -- +4 + +-- !dv_true_v2_false_parallel_true_predicate_6 -- +1 + +-- !dv_true_v2_false_parallel_true_predicate_7 -- +1 + +-- !dv_true_v2_false_parallel_true_predicate_8 -- +1 + +-- !dv_true_v2_false_parallel_true_predicate_9 -- +1 +2 + +-- !dv_true_v2_false_parallel_true_predicate_10 -- +1 + +-- !dv_true_v2_false_parallel_true_predicate_11 -- + +-- !dv_true_v2_false_parallel_true_predicate_12 -- +1 +3 + +-- !dv_true_v2_false_parallel_true_predicate_13 -- +1 +2 + +-- !dv_true_v2_false_parallel_true_predicate_14 -- +3 +4 + +-- !dv_true_v2_false_parallel_true_predicate_15 -- +3 + +-- !dv_true_v2_false_parallel_true_predicate_16 -- +1 +2 +4 + +-- !dv_true_v2_false_parallel_true_predicate_17 -- + +-- !dv_true_v2_false_parallel_true_predicate_18 -- +1 +2 + +-- !dv_true_v2_false_parallel_true_predicate_19 -- +2 +3 +4 + +-- !dv_true_v2_false_parallel_true_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_true_v2_false_parallel_true_native_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_true_native_1 -- +1 +3 + +-- !dv_true_v2_false_parallel_true_native_2 -- +1 +3 + +-- !dv_true_v2_false_parallel_true_native_3 -- + +-- !dv_true_v2_false_parallel_true_native_4 -- + +-- !dv_true_v2_false_parallel_true_native_5 -- +1 + +-- !dv_true_v2_false_parallel_true_preserved_source_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_true_preserved_source_1 -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_true_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_true_v2_false_parallel_true_array_predicate_0 -- +5 + +-- !dv_true_v2_false_parallel_true_array_predicate_1 -- +5 + +-- !dv_true_v2_false_parallel_true_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_true_array_predicate_3 -- +1 + +-- !dv_true_v2_false_parallel_true_array_predicate_4 -- + +-- !dv_true_v2_false_parallel_true_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_true_v2_false_parallel_true_array_predicate_6 -- +5 + +-- !dv_true_v2_false_parallel_true_array_predicate_7 -- +5 + +-- !dv_true_v2_false_parallel_true_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_true_array_predicate_9 -- + +-- !dv_true_v2_false_parallel_true_array_predicate_10 -- +5 +6 + +-- !dv_true_v2_false_parallel_true_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_false_parallel_true_sibling_0 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_sibling_1 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_true_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_true_v2_false_parallel_true_sibling_4 -- +6 elephant \N + +-- !dv_true_v2_false_parallel_true_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_true_sibling_6 -- +6 elephant \N + +-- !dv_true_v2_false_parallel_true_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_true_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_true_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_false_parallel_true_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_sibling_11 -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_true_v2_false_parallel_true_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_true_v2_false_parallel_true_sibling_limit -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_sibling_docvalue_control -- +cat + +-- !dv_true_v2_false_parallel_true_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_true_v2_false_parallel_true_sibling_native_and -- +2 ["cat"] cat + +-- !dv_true_v2_false_parallel_true_native_arrays -- +1 +2 +5 + +-- !dv_true_v2_false_parallel_true_limit_0 -- +1 + +-- !dv_true_v2_false_parallel_true_limit_1 -- +4 + +-- !dv_true_v2_false_parallel_true_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_true_v2_false_parallel_true_doris_source_filtered -- +1 +3 + +-- !dv_true_v2_false_parallel_true_doris_source_filtered_count -- +2 + +-- !dv_true_v2_false_parallel_true_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_true_v2_false_parallel_true_doris_source_multi_count -- +4 + +-- !dv_true_v2_false_parallel_true_doris_source_routed -- +1 +2 + +-- !dv_true_v2_false_parallel_true_doris_source_routed_count -- +2 + +-- !dv_true_v2_false_parallel_true_doris_source_filtered_routed -- +1 + +-- !dv_true_v2_false_parallel_true_doris_source_filtered_routed_count -- +1 + +-- !dv_true_v2_false_parallel_true_doris_source_empty -- + +-- !dv_true_v2_false_parallel_true_doris_source_empty_count -- +0 + +-- !dv_true_v2_false_parallel_true_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_true_v2_false_parallel_true_doris_source_unfiltered_count -- +8 + +-- !dv_true_v2_false_parallel_true_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_true_v2_false_parallel_true_doris_source_with_empty_count -- +4 + +-- !dv_true_v2_true_parallel_false_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_true_v2_true_parallel_false_predicate_0 -- +1 + +-- !dv_true_v2_true_parallel_false_predicate_1 -- +1 +2 + +-- !dv_true_v2_true_parallel_false_predicate_2 -- +1 +3 + +-- !dv_true_v2_true_parallel_false_predicate_3 -- +4 + +-- !dv_true_v2_true_parallel_false_predicate_4 -- +1 +2 +3 + +-- !dv_true_v2_true_parallel_false_predicate_5 -- +4 + +-- !dv_true_v2_true_parallel_false_predicate_6 -- +1 + +-- !dv_true_v2_true_parallel_false_predicate_7 -- +1 + +-- !dv_true_v2_true_parallel_false_predicate_8 -- +1 + +-- !dv_true_v2_true_parallel_false_predicate_9 -- +1 +2 + +-- !dv_true_v2_true_parallel_false_predicate_10 -- +1 + +-- !dv_true_v2_true_parallel_false_predicate_11 -- + +-- !dv_true_v2_true_parallel_false_predicate_12 -- +1 +3 + +-- !dv_true_v2_true_parallel_false_predicate_13 -- +1 +2 + +-- !dv_true_v2_true_parallel_false_predicate_14 -- +3 +4 + +-- !dv_true_v2_true_parallel_false_predicate_15 -- +3 + +-- !dv_true_v2_true_parallel_false_predicate_16 -- +1 +2 +4 + +-- !dv_true_v2_true_parallel_false_predicate_17 -- + +-- !dv_true_v2_true_parallel_false_predicate_18 -- +1 +2 + +-- !dv_true_v2_true_parallel_false_predicate_19 -- +2 +3 +4 + +-- !dv_true_v2_true_parallel_false_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_true_v2_true_parallel_false_native_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_false_native_1 -- +1 +3 + +-- !dv_true_v2_true_parallel_false_native_2 -- +1 +3 + +-- !dv_true_v2_true_parallel_false_native_3 -- + +-- !dv_true_v2_true_parallel_false_native_4 -- + +-- !dv_true_v2_true_parallel_false_native_5 -- +1 + +-- !dv_true_v2_true_parallel_false_preserved_source_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_false_preserved_source_1 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_false_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_true_v2_true_parallel_false_array_predicate_0 -- +5 + +-- !dv_true_v2_true_parallel_false_array_predicate_1 -- +5 + +-- !dv_true_v2_true_parallel_false_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_false_array_predicate_3 -- +1 + +-- !dv_true_v2_true_parallel_false_array_predicate_4 -- + +-- !dv_true_v2_true_parallel_false_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_true_v2_true_parallel_false_array_predicate_6 -- +5 + +-- !dv_true_v2_true_parallel_false_array_predicate_7 -- +5 + +-- !dv_true_v2_true_parallel_false_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_false_array_predicate_9 -- + +-- !dv_true_v2_true_parallel_false_array_predicate_10 -- +5 +6 + +-- !dv_true_v2_true_parallel_false_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_false_sibling_0 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_sibling_1 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_false_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_true_v2_true_parallel_false_sibling_4 -- +6 elephant \N + +-- !dv_true_v2_true_parallel_false_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_false_sibling_6 -- +6 elephant \N + +-- !dv_true_v2_true_parallel_false_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_false_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_false_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_false_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_sibling_11 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_true_v2_true_parallel_false_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_true_v2_true_parallel_false_sibling_limit -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_sibling_docvalue_control -- +cat + +-- !dv_true_v2_true_parallel_false_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_true_v2_true_parallel_false_sibling_native_and -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_false_native_arrays -- +1 +2 +5 + +-- !dv_true_v2_true_parallel_false_limit_0 -- +1 + +-- !dv_true_v2_true_parallel_false_limit_1 -- +4 + +-- !dv_true_v2_true_parallel_false_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_true_v2_true_parallel_false_doris_source_filtered -- +1 +3 + +-- !dv_true_v2_true_parallel_false_doris_source_filtered_count -- +2 + +-- !dv_true_v2_true_parallel_false_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_true_v2_true_parallel_false_doris_source_multi_count -- +4 + +-- !dv_true_v2_true_parallel_false_doris_source_routed -- +1 +2 + +-- !dv_true_v2_true_parallel_false_doris_source_routed_count -- +2 + +-- !dv_true_v2_true_parallel_false_doris_source_filtered_routed -- +1 + +-- !dv_true_v2_true_parallel_false_doris_source_filtered_routed_count -- +1 + +-- !dv_true_v2_true_parallel_false_doris_source_empty -- + +-- !dv_true_v2_true_parallel_false_doris_source_empty_count -- +0 + +-- !dv_true_v2_true_parallel_false_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_true_v2_true_parallel_false_doris_source_unfiltered_count -- +8 + +-- !dv_true_v2_true_parallel_false_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_false_doris_source_with_empty_count -- +4 + +-- !dv_true_v2_true_parallel_true_source -- +1 elephant MiXeD \N elephant \N +2 cat plain \N cat \N +3 rhino UPPER sentinel rhino original +4 \N \N other \N \N + +-- !dv_true_v2_true_parallel_true_predicate_0 -- +1 + +-- !dv_true_v2_true_parallel_true_predicate_1 -- +1 +2 + +-- !dv_true_v2_true_parallel_true_predicate_2 -- +1 +3 + +-- !dv_true_v2_true_parallel_true_predicate_3 -- +4 + +-- !dv_true_v2_true_parallel_true_predicate_4 -- +1 +2 +3 + +-- !dv_true_v2_true_parallel_true_predicate_5 -- +4 + +-- !dv_true_v2_true_parallel_true_predicate_6 -- +1 + +-- !dv_true_v2_true_parallel_true_predicate_7 -- +1 + +-- !dv_true_v2_true_parallel_true_predicate_8 -- +1 + +-- !dv_true_v2_true_parallel_true_predicate_9 -- +1 +2 + +-- !dv_true_v2_true_parallel_true_predicate_10 -- +1 + +-- !dv_true_v2_true_parallel_true_predicate_11 -- + +-- !dv_true_v2_true_parallel_true_predicate_12 -- +1 +3 + +-- !dv_true_v2_true_parallel_true_predicate_13 -- +1 +2 + +-- !dv_true_v2_true_parallel_true_predicate_14 -- +3 +4 + +-- !dv_true_v2_true_parallel_true_predicate_15 -- +3 + +-- !dv_true_v2_true_parallel_true_predicate_16 -- +1 +2 +4 + +-- !dv_true_v2_true_parallel_true_predicate_17 -- + +-- !dv_true_v2_true_parallel_true_predicate_18 -- +1 +2 + +-- !dv_true_v2_true_parallel_true_predicate_19 -- +2 +3 +4 + +-- !dv_true_v2_true_parallel_true_selected_values -- +1 cat cat +2 dog dog +3 Lion cat +4 \N \N + +-- !dv_true_v2_true_parallel_true_native_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_true_native_1 -- +1 +3 + +-- !dv_true_v2_true_parallel_true_native_2 -- +1 +3 + +-- !dv_true_v2_true_parallel_true_native_3 -- + +-- !dv_true_v2_true_parallel_true_native_4 -- + +-- !dv_true_v2_true_parallel_true_native_5 -- +1 + +-- !dv_true_v2_true_parallel_true_preserved_source_0 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_true_preserved_source_1 -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_true_arrays -- +1 ["cat","elephant"] ["cat","elephant"] +2 ["cat"] ["cat"] +3 [] [] +4 [null] [null] +5 cat cat +6 elephant elephant + +-- !dv_true_v2_true_parallel_true_array_predicate_0 -- +5 + +-- !dv_true_v2_true_parallel_true_array_predicate_1 -- +5 + +-- !dv_true_v2_true_parallel_true_array_predicate_2 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_true_array_predicate_3 -- +1 + +-- !dv_true_v2_true_parallel_true_array_predicate_4 -- + +-- !dv_true_v2_true_parallel_true_array_predicate_5 -- +1 +2 +3 +4 +5 +6 + +-- !dv_true_v2_true_parallel_true_array_predicate_6 -- +5 + +-- !dv_true_v2_true_parallel_true_array_predicate_7 -- +5 + +-- !dv_true_v2_true_parallel_true_array_predicate_8 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_true_array_predicate_9 -- + +-- !dv_true_v2_true_parallel_true_array_predicate_10 -- +5 +6 + +-- !dv_true_v2_true_parallel_true_array_predicate_11 -- +1 +2 +3 +4 +6 + +-- !dv_true_v2_true_parallel_true_sibling_0 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_sibling_1 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_sibling_2 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_true_sibling_3 -- +1 ["cat","elephant"] ["cat"] + +-- !dv_true_v2_true_parallel_true_sibling_4 -- +6 elephant \N + +-- !dv_true_v2_true_parallel_true_sibling_5 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_true_sibling_6 -- +6 elephant \N + +-- !dv_true_v2_true_parallel_true_sibling_7 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_true_sibling_8 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_true_sibling_9 -- +1 ["cat","elephant"] ["cat"] +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] + +-- !dv_true_v2_true_parallel_true_sibling_10 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_sibling_11 -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_sibling_12 -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] +4 [null] [] +5 cat [null] +6 elephant \N + +-- !dv_true_v2_true_parallel_true_sibling_filter_only -- +1 ["cat","elephant"] + +-- !dv_true_v2_true_parallel_true_sibling_limit -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_sibling_docvalue_control -- +cat + +-- !dv_true_v2_true_parallel_true_sibling_native -- +1 ["cat","elephant"] ["cat"] +2 ["cat"] cat +3 [] ["cat","dog"] + +-- !dv_true_v2_true_parallel_true_sibling_native_and -- +2 ["cat"] cat + +-- !dv_true_v2_true_parallel_true_native_arrays -- +1 +2 +5 + +-- !dv_true_v2_true_parallel_true_limit_0 -- +1 + +-- !dv_true_v2_true_parallel_true_limit_1 -- +4 + +-- !dv_true_v2_true_parallel_true_hidden -- +1 \N +2 \N +3 \N +4 \N + +-- !dv_true_v2_true_parallel_true_doris_source_filtered -- +1 +3 + +-- !dv_true_v2_true_parallel_true_doris_source_filtered_count -- +2 + +-- !dv_true_v2_true_parallel_true_doris_source_multi -- +1 +3 +6 +8 + +-- !dv_true_v2_true_parallel_true_doris_source_multi_count -- +4 + +-- !dv_true_v2_true_parallel_true_doris_source_routed -- +1 +2 + +-- !dv_true_v2_true_parallel_true_doris_source_routed_count -- +2 + +-- !dv_true_v2_true_parallel_true_doris_source_filtered_routed -- +1 + +-- !dv_true_v2_true_parallel_true_doris_source_filtered_routed_count -- +1 + +-- !dv_true_v2_true_parallel_true_doris_source_empty -- + +-- !dv_true_v2_true_parallel_true_doris_source_empty_count -- +0 + +-- !dv_true_v2_true_parallel_true_doris_source_unfiltered -- +1 +2 +3 +4 +5 +6 +7 +8 + +-- !dv_true_v2_true_parallel_true_doris_source_unfiltered_count -- +8 + +-- !dv_true_v2_true_parallel_true_doris_source_with_empty -- +1 +2 +3 +4 + +-- !dv_true_v2_true_parallel_true_doris_source_with_empty_count -- +4 + diff --git a/regression-test/data/external_table_p2/es/test_es_source_value_boundaries.out b/regression-test/data/external_table_p2/es/test_es_source_value_boundaries.out new file mode 100644 index 00000000000000..3c57d2ac5af883 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_source_value_boundaries.out @@ -0,0 +1,649 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_true_v2_false_parallel_false_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_false_parallel_false_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_true_v2_false_parallel_false_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_false_parallel_false_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_false_parallel_false_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_false_parallel_false_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_true_v2_false_parallel_false_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_true_v2_false_parallel_false_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_true_v2_false_parallel_false_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_true_v2_false_parallel_false_mapping_0_0 -- + +-- !dv_true_v2_false_parallel_false_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_1_0 -- + +-- !dv_true_v2_false_parallel_false_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_2_0 -- + +-- !dv_true_v2_false_parallel_false_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_false_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_false_parallel_true_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_true_v2_false_parallel_true_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_false_parallel_true_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_false_parallel_true_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_false_parallel_true_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_true_v2_false_parallel_true_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_true_v2_false_parallel_true_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_true_v2_false_parallel_true_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_true_v2_false_parallel_true_mapping_0_0 -- + +-- !dv_true_v2_false_parallel_true_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_1_0 -- + +-- !dv_true_v2_false_parallel_true_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_2_0 -- + +-- !dv_true_v2_false_parallel_true_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_false_parallel_true_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_true_parallel_false_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_true_v2_true_parallel_false_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_true_parallel_false_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_true_parallel_false_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_true_parallel_false_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_true_v2_true_parallel_false_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_true_v2_true_parallel_false_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_true_v2_true_parallel_false_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_true_v2_true_parallel_false_mapping_0_0 -- + +-- !dv_true_v2_true_parallel_false_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_1_0 -- + +-- !dv_true_v2_true_parallel_false_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_2_0 -- + +-- !dv_true_v2_true_parallel_false_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_false_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_true_parallel_true_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_true_v2_true_parallel_true_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_true_parallel_true_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_true_v2_true_parallel_true_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_true_v2_true_parallel_true_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_true_v2_true_parallel_true_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_true_v2_true_parallel_true_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_true_v2_true_parallel_true_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_true_v2_true_parallel_true_mapping_0_0 -- + +-- !dv_true_v2_true_parallel_true_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_1_0 -- + +-- !dv_true_v2_true_parallel_true_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_2_0 -- + +-- !dv_true_v2_true_parallel_true_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_true_v2_true_parallel_true_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_false_parallel_false_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_false_v2_false_parallel_false_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_false_parallel_false_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_false_parallel_false_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_false_parallel_false_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_false_v2_false_parallel_false_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_false_v2_false_parallel_false_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_false_v2_false_parallel_false_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_false_v2_false_parallel_false_mapping_0_0 -- + +-- !dv_false_v2_false_parallel_false_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_1_0 -- + +-- !dv_false_v2_false_parallel_false_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_2_0 -- + +-- !dv_false_v2_false_parallel_false_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_false_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_false_parallel_true_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_false_v2_false_parallel_true_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_false_parallel_true_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_false_parallel_true_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_false_parallel_true_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_false_v2_false_parallel_true_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_false_v2_false_parallel_true_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_false_v2_false_parallel_true_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_false_v2_false_parallel_true_mapping_0_0 -- + +-- !dv_false_v2_false_parallel_true_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_1_0 -- + +-- !dv_false_v2_false_parallel_true_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_2_0 -- + +-- !dv_false_v2_false_parallel_true_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_false_parallel_true_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_true_parallel_false_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_false_v2_true_parallel_false_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_true_parallel_false_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_true_parallel_false_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_true_parallel_false_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_false_v2_true_parallel_false_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_false_v2_true_parallel_false_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_false_v2_true_parallel_false_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_false_v2_true_parallel_false_mapping_0_0 -- + +-- !dv_false_v2_true_parallel_false_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_1_0 -- + +-- !dv_false_v2_true_parallel_false_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_2_0 -- + +-- !dv_false_v2_true_parallel_false_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_false_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_source_0 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_true_parallel_true_source_1 -- +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +3 THREE [null] \N 1.0000001 1.5 -3 -3 [] [] [null] + +-- !dv_false_v2_true_parallel_true_source_2 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_true_parallel_true_source_3 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] +2 TWO [] \N 3.0 3 3 3 [null] [null] [] +4 FOUR {"x":1} 9223372036854775807 2.5 2.5 3 3 \N \N \N + +-- !dv_false_v2_true_parallel_true_source_4 -- +1 ONE [{"x":1}] -9223372036854775808 \N \N 3 3 [null, null, 1, null] [null, 1, 0, null] [null, "2024-01-01 00:00:00", null] + +-- !dv_false_v2_true_parallel_true_control_0 -- +1 [{"x":1}] +2 [] +3 [null] +4 {"x":1} + +-- !dv_false_v2_true_parallel_true_control_1 -- +1 -9223372036854775808 +2 \N +3 \N +4 9223372036854775807 + +-- !dv_false_v2_true_parallel_true_control_2 -- +1 \N +2 3.0 +3 1.0000001 +4 2.5 + +-- !dv_false_v2_true_parallel_true_control_3 -- +1 3 +2 3 +3 -3 +4 3 + +-- !dv_false_v2_true_parallel_true_mapping_0_0 -- + +-- !dv_false_v2_true_parallel_true_mapping_0_1 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_0_2 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_0_3 -- +1 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_1_0 -- + +-- !dv_false_v2_true_parallel_true_mapping_1_1 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_1_2 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_1_3 -- +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_2_0 -- + +-- !dv_false_v2_true_parallel_true_mapping_2_1 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_2_2 -- +1 VALUE 5 5 +2 VALUE 5 5 + +-- !dv_false_v2_true_parallel_true_mapping_2_3 -- +1 VALUE 5 5 +2 VALUE 5 5 + diff --git a/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy b/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy new file mode 100644 index 00000000000000..49dc1561eb1fe6 --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_epoch_millis", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + request("DELETE", "doris_epoch_millis") + request("PUT", "doris_epoch_millis", [settings: [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [folded: [type: "custom", filter: ["lowercase"]]]]], + mappings: [properties: [row_id: [type: "long"], + a: [type: "keyword", normalizer: "folded"], d: [type: "date"]]]]) + // Projecting normalized a forces source decoding of every d, including numeric JSON doubles. + ["0", "0000000000000", "1.5", new BigDecimal("1.5"), "1000.", "12345", + "-0.000001", new BigDecimal("-1.5"), "-1000.5", new BigDecimal("1677895728000.5"), + null, ["0"], [], [null]].eachWithIndex { value, i -> + request("PUT", "doris_epoch_millis/_doc/${i + 1}", [row_id: i + 1, a: "VALUE", d: value]) + } + request("POST", "doris_epoch_millis/_refresh") + + sql "drop table if exists es_epoch_millis_expected" + sql """create table es_epoch_millis_expected (row_id bigint,a string,d datetimev2(0)) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_epoch_millis_expected values + (1,'VALUE','1970-01-01 00:00:00'),(2,'VALUE','1970-01-01 00:00:00'), + (3,'VALUE','1970-01-01 00:00:00'),(4,'VALUE','1970-01-01 00:00:00'), + (5,'VALUE','1970-01-01 00:00:01'),(6,'VALUE','1970-01-01 00:00:12'), + (7,'VALUE','1969-12-31 23:59:59'),(8,'VALUE','1969-12-31 23:59:59'), + (9,'VALUE','1969-12-31 23:59:58'),(10,'VALUE','2023-03-04 02:08:48'), + (11,'VALUE',null),(12,'VALUE','1970-01-01 00:00:00'),(13,'VALUE',null),(14,'VALUE',null)""" + def expected = "internal.${context.dbName}.es_epoch_millis_expected" + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalZone = sql("show variables like 'time_zone'")[0][1] + try { + [true, false].each { docvalues -> + def catalog = "es_epoch_millis_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties("type"="es","hosts"="${endpoint}", + "nodes_discovery"="false","enable_docvalue_scan"="${docvalues}")""" + sql "switch ${catalog}" + [false, true].each { scanner -> + sql "set enable_file_scanner_v2 = ${scanner}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + [0, 8].each { hours -> + sql "set time_zone = '+0${hours}:00'" + def prefix = "dv_${docvalues}_v2_${scanner}_parallel_${parallel}_zone_${hours}" + def oracle = "(select row_id,a,date_add(d,interval ${hours} hour) as d from ${expected}) e" + ["true", "d is null", "d < '1970-01-01 00:00:00'", + "d = '1970-01-01 08:00:00'"].eachWithIndex { predicate, i -> + def query = "select row_id,a,d from doris_epoch_millis where ${predicate} order by row_id" + quickTest("${prefix}_source_${i}", query, + "select row_id,a,d from ${oracle} where ${predicate} order by row_id") + quickTest("${prefix}_source_${i}", query) + } + def control = "select row_id,d from doris_epoch_millis order by row_id" + quickTest("${prefix}_control", control, + "select row_id,d from ${oracle} order by row_id") + quickTest("${prefix}_control", control) + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalZone}'" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy b/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy new file mode 100644 index 00000000000000..d3ff3bb8e269b7 --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_mapping_boundaries", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def settings = [number_of_shards: 1, number_of_replicas: 0] + def populated = "doris_mapping_boundary_m" + request("DELETE", populated) + request("PUT", populated, [settings: settings, mappings: [properties: [ + row_id: [type: "long"], amount: [type: "long", index: false, doc_values: true], + event_time: [type: "date", format: "epoch_millis", index: false], + flag: [type: "boolean", index: false], tag: [type: "keyword"]]]]) + [[row_id: 1, amount: 1, event_time: 1700000000000, flag: true, tag: "one"], + [row_id: 2, amount: 2, event_time: 1700000001000, flag: false, tag: "ok"], + [row_id: 3, amount: 3, event_time: 1700000002000, flag: true, tag: "other"], + [row_id: 4, amount: null, event_time: null, flag: false, tag: "none"]].each { row -> + request("PUT", "${populated}/_doc/${row.row_id}", row) + } + request("POST", "${populated}/_refresh") + def aliases = [] + [[enabled: false], [date_detection: false], [numeric_detection: true], + [dynamic_date_formats: ["yyyy-MM-dd"]], [_source: [enabled: false]], [dynamic: false]].eachWithIndex { config, i -> + ["a", "z"].each { order -> + def backing = "doris_mapping_boundary_${order}_${i}" + def alias = "doris_mapping_boundary_alias_${order}_${i}" + request("DELETE", backing) + request("PUT", backing, [settings: settings, mappings: config]) + def sourceOnly = config.enabled == false || config.dynamic == false + if (sourceOnly) { + // Disabled parsing still stores source documents, which alias scans must retain. + request("PUT", "${backing}/_doc/99?refresh=true", [row_id: 99, amount: 99]) + } + request("POST", "_aliases", [actions: [ + [add: [index: populated, alias: alias]], [add: [index: backing, alias: alias]]]]) + aliases.add([name: alias, sourceOnly: sourceOnly]) + } + } + sql "drop table if exists es_mapping_boundary_expected" + sql """create table es_mapping_boundary_expected ( + row_id bigint, amount bigint, event_time bigint, flag boolean, tag string + ) distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_mapping_boundary_expected values + (1,1,1700000000000,true,'one'),(2,2,1700000001000,false,'ok'), + (3,3,1700000002000,true,'other'),(4,null,null,false,'none')""" + def expected = "internal.${context.dbName}.es_mapping_boundary_expected" + def nativeTerm = """esquery(tag, '{"term":{"tag":"ok"}}')""" + def predicates = ["amount = 1 or ${nativeTerm}", "not (amount = 1 or ${nativeTerm})", + "amount = 1 and ${nativeTerm}", "amount in (1,3) or ${nativeTerm}", + "amount > 1 or ${nativeTerm}", "flag = true or ${nativeTerm}", + "not (flag = true or ${nativeTerm})", "event_time = 1700000000000 or ${nativeTerm}", + "not (event_time = 1700000000000 or ${nativeTerm})", "event_time > 1700000000000 or ${nativeTerm}"] + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + try { + [true, false].each { docvalues -> + def catalog = "es_mapping_boundary_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + "type"="es", "hosts"="${endpoint}", "nodes_discovery"="false", + "enable_docvalue_scan"="${docvalues}" + )""" + sql "switch ${catalog}" + [false, true].each { scannerV2 -> + sql "set enable_file_scanner_v2 = ${scannerV2}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + def prefix = "dv_${docvalues}_v2_${scannerV2}_parallel_${parallel}" + predicates.eachWithIndex { predicate, i -> + def query = "select row_id, amount, event_time, flag from ${populated} " + + "where ${predicate} order by row_id" + quickTest("${prefix}_native_${i}", query, + "select row_id, amount, event_time, flag from ${expected} " + + "where ${predicate.replace(nativeTerm, "tag = 'ok'")} order by row_id") + quickTest("${prefix}_native_${i}", query) + } + aliases.eachWithIndex { alias, i -> + def query = "select row_id, amount from ${alias.name} order by row_id" + def oracle = "select row_id, amount from ${expected}" + + (alias.sourceOnly ? " union all select cast(99 as bigint), cast(99 as bigint)" : "") + quickTest("${prefix}_alias_${i}", query, "select * from (${oracle}) t order by row_id") + quickTest("${prefix}_alias_${i}", query) + // Fields missing from a configuration-only backing cannot narrow source hits. + def filterQuery = "select row_id from ${alias.name} where amount = 99 order by row_id" + quickTest("${prefix}_alias_filter_${i}", filterQuery, + "select cast(99 as bigint)" + (alias.sourceOnly ? "" : " where false")) + quickTest("${prefix}_alias_filter_${i}", filterQuery) + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy new file mode 100644 index 00000000000000..5571132290d275 --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_numeric_source_semantics", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def index = "doris_numeric_source_safety" + request("DELETE", index) + request("PUT", index, [ + settings: [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [lowercase_value: [type: "custom", filter: ["lowercase"]]]]], + mappings: [properties: [row_id: [type: "long"], + a: [type: "keyword", normalizer: "lowercase_value"], + b: [type: "scaled_float", scaling_factor: 4], c: [type: "half_float"]]]]) + [[row_id: 1, a: "ok", b: 2.34, c: 2.34], [row_id: 2, a: "empty", b: null, c: null], + [row_id: 3, a: "other", b: -2.34, c: -2.34]].each { row -> + request("PUT", "${index}/_doc/${row.row_id}", row) + } + request("POST", "${index}/_refresh") + + sql "drop table if exists es_numeric_source_expected" + sql """create table es_numeric_source_expected (row_id bigint, a string, b double, c float) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_numeric_source_expected values + (1,'ok',2.34,2.34),(2,'empty',null,null),(3,'other',-2.34,-2.34)""" + def expectedTable = "internal.${context.dbName}.es_numeric_source_expected" + def predicates = ["b = 2.25", "b = 2.34", "b > 2.25", "b < 2.30", "b != 2.25", + "b in (2.34, -2.34)", "b not in (2.25)", "not (b = 2.25)", "b is null", "b is not null", + "b = 2.25 or row_id = 3", "not (b > 2.25 and row_id = 1)", + "c = cast(2.34 as float)", "c = cast(2.33984375 as float)", + "c > cast(2.33984375 as float)", "c in (cast(2.34 as float), cast(-2.34 as float))"] + def nativeTerm = """esquery(b, '{"term":{"b":2.25}}')""" + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + try { + ["enabled", "disabled", "limited"].each { mode -> + def catalog = "es_numeric_source_${mode}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + "type"="es", "hosts"="${endpoint}", "nodes_discovery"="false", + "enable_docvalue_scan"="${mode != 'disabled'}", + "max_docvalue_fields"="${mode == 'limited' ? 1 : 20}" + )""" + sql "switch ${catalog}" + [false, true].each { scannerV2 -> + sql "set enable_file_scanner_v2 = ${scannerV2}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + def tag = "${mode}_v2_${scannerV2}_parallel_${parallel}" + // The normalizer forces the whole projection to source. Numeric predicates must + // compare the original values, even when the indexed representation is rounded. + predicates.eachWithIndex { predicate, i -> + def query = "select row_id, a, b, c from ${index} where ${predicate} order by row_id" + quickTest("${tag}_predicate_${i}", query, + "select row_id, a, b, c from ${expectedTable} where ${predicate} order by row_id") + quickTest("${tag}_predicate_${i}", query) + } + def filterOnly = "select a from ${index} where b = 2.25 order by a" + quickTest("${tag}_filter_only", filterOnly, + "select a from ${expectedTable} where b = 2.25 order by a") + quickTest("${tag}_filter_only", filterOnly) + def nativeQuery = "select a, b from ${index} where ${nativeTerm}" + quickTest("${tag}_native", nativeQuery, "select a, b from ${expectedTable} where row_id = 1") + quickTest("${tag}_native", nativeQuery) + def nativeAnd = "select a, b from ${index} where b = 2.25 and ${nativeTerm}" + quickTest("${tag}_native_and", nativeAnd, + "select a, b from ${expectedTable} where false") + quickTest("${tag}_native_and", nativeAnd) + test { + sql "select a, b from ${index} where b = 2.25 or ${nativeTerm}" + exception "Cannot evaluate esquery locally" + } + // A numeric-only projection retains docvalues and its existing indexed-value + // semantics. This control prevents fixing source predicates by disabling docvalues. + ["b": "2.25", "c": "2.33984375"].each { column, rounded -> + def query = "select ${column} from ${index} where ${column} = ${rounded}" + def oracle = "select cast(${rounded} as ${column == 'b' ? 'double' : 'float'})" + + (mode == "disabled" ? " where false" : "") + quickTest("${tag}_${column}_docvalue_control", query, oracle) + quickTest("${tag}_${column}_docvalue_control", query) + } + // With no normalized column projected, only the field limit or catalog flag + // forces source mode; the same numeric guard must apply to both fallbacks. + def numericOnly = "select row_id, b, c from ${index} where b = 2.25" + def numericOracle = "select cast(1 as bigint), cast(2.25 as double), " + + "cast(2.33984375 as float)" + (mode == "enabled" ? "" : " where false") + quickTest("${tag}_numeric_projection", numericOnly, numericOracle) + quickTest("${tag}_numeric_projection", numericOnly) + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy b/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy new file mode 100644 index 00000000000000..a0616b5b6d619b --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_scalar_materialization", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def settings = [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [folded: [type: "custom", filter: ["lowercase"]]]]] + request("DELETE", "doris_scalar_materialization") + request("PUT", "doris_scalar_materialization", [settings: settings, mappings: [ + _meta: [doris: [array_fields: ["narr"]]], properties: [row_id: [type: "long"], + a: [type: "keyword", normalizer: "folded"], f: [type: "float"], d: [type: "double"], + truth: [type: "boolean"], day: [type: "date"], n: [type: "byte"], s: [type: "short"], + i: [type: "integer"], l: [type: "long"], narr: [type: "byte"]]]]) + [[row_id: 1, a: "one", f: [1.5], d: [2.5], truth: [""], day: ["2024-01-01"], + n: "0.99999999999999999", s: "0.99999999999999999", i: "0.99999999999999999", + l: "0.99999999999999999", narr: ["0.99999999999999999", "-0.99999999999999999"]], + [row_id: 2, a: "two", f: [], d: [], truth: [], day: [], n: "-0.99999999999999999", + s: "-0.99999999999999999", i: "-0.99999999999999999", l: "-0.99999999999999999", narr: []], + [row_id: 3, a: "nullish", f: [null], d: [null], truth: [null], day: [null]], + [row_id: 4, a: "missing"], + [row_id: 5, a: "scalar", f: 3.5, d: 4.5, truth: true, day: "2024-01-01", + n: "127.9", s: 32767.9, i: 2147483647, l: "9223372036854775807", narr: [127]], + [row_id: 6, a: "negative", f: "1.25", d: "2.25", truth: "true", day: "2024-01-01T08:00:00+08:00", + n: "-128.9", s: "-32768", i: "-2147483648", l: "-9223372036854775808", narr: [-128]], + [row_id: 7, a: "nan", n: "NaN", s: "NaN", i: "NaN", narr: ["NaN"]]].each { row -> + request("PUT", "doris_scalar_materialization/_doc/${row.row_id}", row) + } + request("POST", "doris_scalar_materialization/_refresh") + request("DELETE", "doris_retained_range_not") + request("PUT", "doris_retained_range_not", [settings: settings, mappings: [properties: [ + row_id: [type: "long"], n: [type: "long"], b: [type: "keyword"]]]]) + [[row_id: 1, n: [1,2], b: "miss"], [row_id: 2, n: [2,3], b: "miss"], + [row_id: 3, n: [3], b: "miss"], [row_id: 4, n: null, b: "miss"], + [row_id: 5, n: [1,2], b: "hit"]].each { row -> + request("PUT", "doris_retained_range_not/_doc/${row.row_id}", row) + } + request("POST", "doris_retained_range_not/_refresh") + sql "drop table if exists es_scalar_materialization_expected" + sql """create table es_scalar_materialization_expected ( + row_id bigint,a string,f float,d double,truth boolean,day datetimev2(0),n tinyint,s smallint, + i int,l bigint,narr array + ) distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_scalar_materialization_expected values + (1,'one',1.5,2.5,false,'2024-01-01 00:00:00',1,1,1,0,[1,-1]), + (2,'two',null,null,null,null,-1,-1,-1,0,[]), + (3,'nullish',null,null,null,null,null,null,null,null,null), + (4,'missing',null,null,null,null,null,null,null,null,null), + (5,'scalar',3.5,4.5,true,'2024-01-01 00:00:00',127,32767,2147483647,9223372036854775807,[127]), + (6,'negative',1.25,2.25,true,'2024-01-01 08:00:00',-128,-32768,-2147483648,-9223372036854775808,[-128]), + (7,'nan',null,null,null,null,0,0,0,null,[0])""" + def expected = "internal.${context.dbName}.es_scalar_materialization_expected" + sql "drop table if exists es_retained_range_expected" + sql """create table es_retained_range_expected (row_id bigint,n bigint,b string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into es_retained_range_expected values (1,1,'miss'),(2,2,'miss'),(3,3,'miss'),(4,null,'miss'),(5,1,'hit')" + def rangeExpected = "internal.${context.dbName}.es_retained_range_expected" + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalZone = sql("show variables like 'time_zone'")[0][1] + def originalRules = sql("show variables like 'disable_nereids_expression_rules'")[0][1] + try { + sql "set time_zone = 'Asia/Shanghai'" + sql "set disable_nereids_expression_rules = 'SIMPLIFY_NOT_EXPR'" + [true, false].each { docvalues -> + def catalog = "es_scalar_materialization_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties("type"="es","hosts"="${endpoint}", + "nodes_discovery"="false","enable_docvalue_scan"="${docvalues}")""" + sql "switch ${catalog}" + [false, true].each { scanner -> + sql "set enable_file_scanner_v2 = ${scanner}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + def prefix = "dv_${docvalues}_v2_${scanner}_parallel_${parallel}" + def predicates = ["true", "f is null", "d is not null", "truth is null", "day is not null", + "n = 1", "s = 1", "i = 1", "n < 0", "s < 0", "i < 0", "l = 0"] + predicates.eachWithIndex { predicate, j -> + def projection = "row_id,a,f,d,truth,day,n,s,i,l,narr" + def query = "select ${projection} from doris_scalar_materialization where ${predicate} order by row_id" + quickTest("${prefix}_source_${j}", query, + "select ${projection} from ${expected} where ${predicate} order by row_id") + quickTest("${prefix}_source_${j}", query) + } + ["f", "d", "truth", "n", "s", "i", "l"].each { column -> + def query = "select row_id,${column} from doris_scalar_materialization order by row_id" + quickTest("${prefix}_${column}_control", query, + "select row_id,${column} from ${expected} order by row_id") + quickTest("${prefix}_${column}_control", query) + } + if (docvalues) { + // Numeric docvalues select the first sorted value. These LT/LE complements + // exercise arrays spanning the threshold, scalar controls and missing values. + ["n < 2", "n <= 2"].eachWithIndex { predicate, j -> + def query = "select row_id,n from doris_retained_range_not where not(${predicate}) " + + "or esquery(b, '{\"term\":{\"b\":\"hit\"}}') order by row_id" + quickTest("${prefix}_range_not_${j}", query, + "select row_id,n from ${rangeExpected} where not(${predicate}) or b = 'hit' order by row_id") + quickTest("${prefix}_range_not_${j}", query) + } + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalZone}'" + sql "set disable_nereids_expression_rules = '${originalRules}'" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy b/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy new file mode 100644 index 00000000000000..034bbf6a995f2d --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_source_fallback_types", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def settings = [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [lowercase_value: [type: "custom", filter: ["lowercase"]]]]] + def index = "doris_source_fallback_types" + def properties = [row_id: [type: "long"], a: [type: "keyword", normalizer: "lowercase_value"], + n: [type: "long"], alias_value: [type: "alias", path: "n"], + generated_value: [type: "long", script: [source: "emit(5)"]], + items: [type: "long"], event_time: [type: "date"], truth: [type: "boolean"], flags: [type: "boolean"]] + request("DELETE", index) + request("PUT", index, [settings: settings, mappings: [ + _meta: [doris: [array_fields: ["items", "flags"]]], properties: properties]]) + [[row_id: 1, a: "ok", n: 5, items: [], event_time: "2024-01-01", truth: "", flags: ["", true, false]], + [row_id: 2, a: "other", n: 7, items: [1,2], event_time: "2024-01-01T08:00:00+08:00", + truth: false, flags: [true]], + [row_id: 3, a: "nullish", n: null, items: null, event_time: null, truth: "false", flags: []], + [row_id: 4, a: "absent", n: 0, truth: true, flags: null]].each { row -> + request("PUT", "${index}/_doc/${row.row_id}", row) + } + request("POST", "${index}/_refresh") + ["flag", "mode"].each { kind -> + def disabled = "doris_source_disabled_${kind}" + request("DELETE", disabled) + request("PUT", disabled, [settings: settings, mappings: [ + _source: kind == "flag" ? [enabled: false] : [mode: "disabled"], + properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]]) + request("PUT", "${disabled}/_doc/1?refresh=true", [row_id: 1, a: "ok", n: 1]) + request("POST", "_aliases", [actions: [[add: [index: index, alias: "doris_source_disabled_alias_${kind}"]], + [add: [index: disabled, alias: "doris_source_disabled_alias_${kind}"]]]]) + } + request("DELETE", "doris_source_root_disabled") + request("PUT", "doris_source_root_disabled", [settings: settings, mappings: [enabled: false, + properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]]) + request("PUT", "doris_source_root_disabled/_doc/1?refresh=true", [row_id: 1, a: "ok", n: 1]) + request("POST", "_aliases", [actions: [[add: [index: index, alias: "doris_source_root_alias"]], + [add: [index: "doris_source_root_disabled", alias: "doris_source_root_alias"]]]]) + request("DELETE", "doris_source_integer_coercion") + request("PUT", "doris_source_integer_coercion", [settings: settings, mappings: [ + properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]]) + [[row_id: 1, a: "exponent", n: "1.2e3"], [row_id: 2, a: "singleton", n: [5]], + [row_id: 3, a: "negative", n: "-1.9E1"], [row_id: 4, a: "maximum", n: "9223372036854775807"], + [row_id: 5, a: "fraction", n: "0.9"]].each { row -> + request("PUT", "doris_source_integer_coercion/_doc/${row.row_id}", row) + } + request("POST", "doris_source_integer_coercion/_refresh") + request("DELETE", "doris_retained_not") + request("PUT", "doris_retained_not", [settings: settings, mappings: [ + properties: [row_id: properties.row_id, n: properties.n, b: [type: "keyword"]]]]) + [[row_id: 1, n: null, b: "miss"], [row_id: 2, n: 1, b: "miss"], + [row_id: 3, n: 2, b: "hit"], [row_id: 4, n: 2, b: "miss"]].each { row -> + request("PUT", "doris_retained_not/_doc/${row.row_id}", row) + } + request("POST", "doris_retained_not/_refresh") + sql "drop table if exists es_source_fallback_expected" + sql """create table es_source_fallback_expected ( + row_id bigint, a string, n bigint, alias_value bigint, generated_value bigint, + items array, event_time datetimev2(0), truth boolean, flags array + ) distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_source_fallback_expected values + (1,'ok',5,null,null,[],'2024-01-01 00:00:00',false,[false,true,false]), + (2,'other',7,null,null,[1,2],'2024-01-01 08:00:00',false,[true]), + (3,'nullish',null,null,null,null,null,false,[]), + (4,'absent',0,null,null,null,null,true,null)""" + def expected = "internal.${context.dbName}.es_source_fallback_expected" + def queries = [ + ["row_id,a,alias_value", "alias_value = 5"], ["row_id,a,alias_value", "alias_value is null"], + ["row_id,a,generated_value", "generated_value = 5"], ["row_id,a,generated_value", "generated_value is null"], + ["row_id,a,items", "items is null"], ["row_id,a,items", "items is not null"], + ["row_id,a,event_time", "event_time = cast('2024-01-01 08:00:00' as datetimev2(0))"], + ["row_id,a,event_time", "event_time = cast('2024-01-01 00:00:00' as datetimev2(0))"], + ["row_id,a,truth,flags", "row_id > 0"], ["row_id,a,truth", "truth = false"]] + sql "drop table if exists es_integer_coercion_expected" + sql """create table es_integer_coercion_expected (row_id bigint,a string,n bigint) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_integer_coercion_expected values (1,'exponent',1200),(2,'singleton',5), + (3,'negative',-19),(4,'maximum',9223372036854775807),(5,'fraction',0)""" + def integerExpected = "internal.${context.dbName}.es_integer_coercion_expected" + sql "drop table if exists es_retained_not_expected" + sql """create table es_retained_not_expected (row_id bigint,n bigint,b string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into es_retained_not_expected values (1,null,'miss'),(2,1,'miss'),(3,2,'hit'),(4,2,'miss')" + def notExpected = "internal.${context.dbName}.es_retained_not_expected" + def originalNotRules = sql("show variables like 'disable_nereids_expression_rules'")[0][1] + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalZone = sql("show variables like 'time_zone'")[0][1] + try { + sql "set time_zone = 'Asia/Shanghai'" + sql "set disable_nereids_expression_rules = 'SIMPLIFY_NOT_EXPR'" + [true, false].each { docvalues -> + def catalog = "es_source_fallback_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + "type"="es", "hosts"="${endpoint}", "nodes_discovery"="false", + "enable_docvalue_scan"="${docvalues}" + )""" + sql "switch ${catalog}" + [false, true].each { scannerV2 -> + sql "set enable_file_scanner_v2 = ${scannerV2}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + def prefix = "dv_${docvalues}_v2_${scannerV2}_parallel_${parallel}" + queries.eachWithIndex { pair, i -> + def query = "select ${pair[0]} from ${index} where ${pair[1]} order by row_id" + quickTest("${prefix}_source_${i}", query, + "select ${pair[0]} from ${expected} where ${pair[1]} order by row_id") + quickTest("${prefix}_source_${i}", query) + } + ["alias_value", "generated_value"].each { column -> + def query = "select row_id,${column} from ${index} where ${column} = 5 order by row_id" + def predicate = column == "alias_value" ? "n = 5" : "row_id > 0" + def oracle = "select row_id,cast(5 as bigint) from ${expected} where " + + (docvalues ? predicate : "false") + " order by row_id" + quickTest("${prefix}_${column}_control", query, oracle) + quickTest("${prefix}_${column}_control", query) + } + // Check date read-mode differences without depending on the FE JVM timezone + // used by the existing docvalue date-literal pushdown conversion. + def dateControl = "select row_id,event_time from ${index} where event_time is not null order by row_id" + def dateProjection = docvalues ? "cast('2024-01-01 08:00:00' as datetimev2(0))" : "event_time" + def dateOracle = "select row_id,${dateProjection} from ${expected} " + + "where event_time is not null order by row_id" + quickTest("${prefix}_date_control", dateControl, dateOracle) + quickTest("${prefix}_date_control", dateControl) + def boolControl = "select truth from ${index} where row_id = 1" + quickTest("${prefix}_boolean_control", boolControl, "select cast(false as boolean)") + quickTest("${prefix}_boolean_control", boolControl) + ["true", "n = 1200", "n = 5", "n < 0"].eachWithIndex { predicate, i -> + def query = "select row_id,a,n from doris_source_integer_coercion where ${predicate} order by row_id" + quickTest("${prefix}_integer_${i}", query, + "select row_id,a,n from ${integerExpected} where ${predicate} order by row_id") + quickTest("${prefix}_integer_${i}", query) + def control = "select row_id,n from doris_source_integer_coercion where ${predicate} order by row_id" + quickTest("${prefix}_integer_control_${i}", control, + "select row_id,n from ${integerExpected} where ${predicate} order by row_id") + quickTest("${prefix}_integer_control_${i}", control) + } + ["not(n <> 1)", "not(n = 1)", "not(not(n <> 1))", "not(n < 1 or n > 2)"].eachWithIndex { predicate, i -> + def query = "select row_id,n from doris_retained_not where ${predicate} " + + "or esquery(b, '{\"term\":{\"b\":\"hit\"}}') order by row_id" + quickTest("${prefix}_retained_not_${i}", query, + "select row_id,n from ${notExpected} where ${predicate} or b = 'hit' order by row_id") + quickTest("${prefix}_retained_not_${i}", query) + } + ["doris_source_root_disabled", "doris_source_root_alias"].eachWithIndex { target, i -> + ["n is null", "n = 1"].eachWithIndex { predicate, j -> + def query = "select a,n from ${target} where ${predicate} order by a" + def oracle = "select a,n from (select 'ok' as a,cast(1 as bigint) as n) r where ${predicate}" + if (i == 1) { + oracle = "select a,n from (${oracle} union all select a,n from ${expected} " + + "where ${predicate}) t order by a" + } + quickTest("${prefix}_root_${i}_${j}", query, oracle) + quickTest("${prefix}_root_${i}_${j}", query) + } + } + ["flag", "mode"].each { kind -> + ["doris_source_disabled_${kind}", "doris_source_disabled_alias_${kind}"].eachWithIndex { target, i -> + def query = "select a,n from ${target} where n = 1" + quickTest("${prefix}_disabled_${kind}_${i}", query, + "select cast(null as string),cast(null as bigint) where false") + quickTest("${prefix}_disabled_${kind}_${i}", query) + def allRows = "select a,n from ${target} order by a" + def oracle = "select cast(null as string) as a,cast(null as bigint) as n" + if (i == 1) { + oracle = "select a,n from (${oracle} union all select a,n from ${expected}) t order by a" + } + quickTest("${prefix}_disabled_rows_${kind}_${i}", allRows, oracle) + quickTest("${prefix}_disabled_rows_${kind}_${i}", allRows) + } + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalZone}'" + sql "set disable_nereids_expression_rules = '${originalNotRules}'" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy b/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy new file mode 100644 index 00000000000000..831636f700374d --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_source_semantics", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def properties = [ + row_id: [type: "long"], tenant: [type: "keyword"], + limited: [type: "keyword", ignore_above: 4, doc_values: true], + normalized: [type: "keyword", normalizer: "lowercase_value"], + nullable_value: [type: "keyword", null_value: "sentinel"], + body: [type: "text", fields: [raw: [type: "keyword", ignore_above: 4]]], + copy_input: [type: "keyword", copy_to: "copied"], copied: [type: "keyword"], + multi_body: [type: "text", fields: [ + aa_folded: [type: "keyword", normalizer: "lowercase_value"], + zz_raw: [type: "keyword", doc_values: true]]], + doconly: [type: "keyword", index: false, doc_values: true] + ] + def settings = [number_of_shards: 2, number_of_replicas: 0, + analysis: [normalizer: [lowercase_value: [type: "custom", filter: ["lowercase"]]]]] + ["doris_source_safety_a", "doris_source_safety_b", "doris_source_safety_hidden", + "doris_source_safety_included"].each { index -> + request("DELETE", index) + def mappings = [properties: properties] + if (index.endsWith("hidden")) { + mappings._source = [excludes: ["limit*"]] + } else if (index.endsWith("included")) { + mappings._source = [includes: ["row_*", "tenant", "multi_*", "doconly"]] + } + request("PUT", index, [settings: settings, mappings: mappings]) + } + + request("DELETE", "doris_source_safety_z_empty") + request("PUT", "doris_source_safety_z_empty", [settings: [number_of_shards: 2, number_of_replicas: 0]]) + + // Use distinct shards so the routing-alias oracle cannot pass because two routing keys collide. + def routingKeys = (0..<32).collect { "route_${it}" } + def firstRouting = routingKeys[0] + def firstShard = request("GET", "doris_source_safety_a/_search_shards?routing=${firstRouting}").shards[0][0].shard + def secondRouting = routingKeys.find { + request("GET", "doris_source_safety_a/_search_shards?routing=${it}").shards[0][0].shard != firstShard + } + assertTrue(secondRouting != null, "fixture requires two distinct routing shards") + + def sourceRows = [ + [row_id: 1, tenant: "A", limited: "elephant", normalized: "MiXeD", nullable_value: null, + body: "elephant", copy_input: "copy-one", copied: null, multi_body: "cat", doconly: "cat"], + [row_id: 2, tenant: "B", limited: "cat", normalized: "plain", body: "cat", multi_body: "dog", doconly: "dog"], + [row_id: 3, tenant: "A", limited: "rhino", normalized: "UPPER", nullable_value: "sentinel", + body: "rhino", copied: "original", multi_body: "Lion", doconly: "cat"], + [row_id: 4, tenant: "B", limited: null, normalized: null, nullable_value: "other"] + ] + ["doris_source_safety_a", "doris_source_safety_b", "doris_source_safety_hidden", + "doris_source_safety_included"].each { index -> + sourceRows.eachWithIndex { row, offset -> + def document = new LinkedHashMap(row) + if (index.endsWith("_b")) { + document.row_id += 4 + } + def routing = offset < 2 ? firstRouting : secondRouting + request("PUT", "${index}/_doc/${document.row_id}?routing=${routing}", document) + } + request("POST", "${index}/_refresh") + } + request("POST", "_aliases", [actions: [ + [add: [index: "doris_source_safety_a", alias: "doris_source_with_empty"]], + [add: [index: "doris_source_safety_z_empty", alias: "doris_source_with_empty"]], + [add: [index: "doris_source_safety_a", alias: "doris_source_filtered", filter: [term: [tenant: "A"]]]], + [add: [index: "doris_source_safety_a", alias: "doris_source_multi", filter: [term: [tenant: "A"]]]], + [add: [index: "doris_source_safety_b", alias: "doris_source_multi", filter: [term: [tenant: "B"]]]], + [add: [index: "doris_source_safety_a", alias: "doris_source_routed", search_routing: firstRouting]], + [add: [index: "doris_source_safety_a", alias: "doris_source_filtered_routed", + search_routing: firstRouting, filter: [term: [tenant: "A"]]]], + [add: [index: "doris_source_safety_a", alias: "doris_source_empty", filter: [term: [tenant: "absent"]]]], + [add: [index: "doris_source_safety_a", alias: "doris_source_unfiltered"]], + [add: [index: "doris_source_safety_b", alias: "doris_source_unfiltered"]] + ]]) + + sql "drop table if exists es_source_safety_expected" + sql """create table es_source_safety_expected ( + row_id bigint, tenant string, limited string, normalized string, nullable_value string, + body string, copy_input string, copied string, multi_body string, doconly string + ) distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_source_safety_expected values + (1,'A','elephant','MiXeD',NULL,'elephant','copy-one',NULL,'cat','cat'), + (2,'B','cat','plain',NULL,'cat',NULL,NULL,'dog','dog'), + (3,'A','rhino','UPPER','sentinel','rhino',NULL,'original','Lion','cat'), + (4,'B',NULL,NULL,'other',NULL,NULL,NULL,NULL,NULL)""" + def expectedTable = "internal.${context.dbName}.es_source_safety_expected" + def predicates = [ + "limited = 'elephant'", "limited in ('elephant', 'cat')", "limited != 'cat'", + "limited is null", "limited is not null", "limited <=> null", + "limited like 'ele%'", "limited regexp 'ele.*'", + "body = 'elephant'", "body in ('elephant', 'cat')", + "normalized = 'MiXeD'", "normalized = 'mixed'", "normalized < 'Z'", + "nullable_value is null", "nullable_value is not null", "nullable_value = 'sentinel'", + "copied is null", "copied = 'copy-one'", + "(limited = 'elephant' and tenant = 'A') or row_id = 2", + "not (limited = 'elephant' and tenant = 'A')" + ] + def nativeTerm = """esquery(tenant, '{"term":{"tenant":"A"}}')""" + def mixedNativePredicates = [ + ["tenant = 'B' or ${nativeTerm}", "tenant = 'B' or tenant = 'A'"], + ["multi_body = 'cat' or ${nativeTerm}", "multi_body = 'cat' or tenant = 'A'"], + ["doconly = 'cat' or ${nativeTerm}", "doconly = 'cat' or tenant = 'A'"], + ["not (tenant = 'B' or ${nativeTerm})", "not (tenant = 'B' or tenant = 'A')"], + ["limited = 'elephant' and not (${nativeTerm})", "limited = 'elephant' and not (tenant = 'A')"], + ["limited = 'elephant' and ${nativeTerm}", "limited = 'elephant' and tenant = 'A'"] + ] + request("DELETE", "doris_source_safety_arrays") + request("PUT", "doris_source_safety_arrays", [settings: [number_of_shards: 2, number_of_replicas: 0], + mappings: [properties: [row_id: properties.row_id, limited: properties.limited, body: properties.body, + sibling: [type: "keyword"]]]]) + def arrayValues = [["cat", "elephant"], ["cat"], [], [null], "cat", "elephant"] + def siblingValues = [["cat"], "cat", ["cat", "dog"], [], [null], null] + arrayValues.eachWithIndex { value, i -> + request("PUT", "doris_source_safety_arrays/_doc/${i + 1}", + [row_id: i + 1, limited: value, body: value, sibling: siblingValues[i]]) + } + request("POST", "doris_source_safety_arrays/_refresh") + sql "drop table if exists es_source_array_expected" + sql """create table es_source_array_expected (row_id bigint, limited string, body string, sibling string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + def arrayRows = arrayValues.withIndex().collect { value, i -> + String text = value instanceof List ? JsonOutput.toJson(value) : value + def sibling = siblingValues[i] + String siblingSql = sibling == null ? "NULL" + : "'${sibling instanceof List ? JsonOutput.toJson(sibling) : sibling}'" + "(${i + 1},'${text}','${text}',${siblingSql})" + }.join(",") + sql "insert into es_source_array_expected values ${arrayRows}" + def arrayExpectedTable = "internal.${context.dbName}.es_source_array_expected" + def arrayPredicates = ["limited = 'cat'", "limited in ('cat', 'four')", "limited != 'cat'", + "limited = '[\"cat\",\"elephant\"]'", "limited is null", "limited is not null", + "body = 'cat'", "body in ('cat', 'four')", "body != 'cat'", "body is null", + "limited = 'cat' or body = 'elephant'", "not (limited = 'cat' and body = 'cat')"] + def siblingPredicates = ["sibling = 'cat'", "sibling in ('cat', 'dog')", "sibling != 'cat'", + "sibling = '[\"cat\"]'", "sibling is null", "sibling is not null", "sibling <=> null", + "sibling like '[%'", "sibling not in ('cat')", "not (sibling = 'cat')", + "sibling = 'cat' or row_id = 1", "sibling = 'cat' and row_id > 0", + "not (sibling = 'cat' and row_id = 1)"] + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalBatch = sql("show variables like 'batch_size'")[0][1] + try { + [false, true].each { docvalues -> + def catalog = "es_source_safety_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + "type"="es", "hosts"="${endpoint}", "nodes_discovery"="false", + "enable_keyword_sniff"="true", "enable_docvalue_scan"="${docvalues}" + )""" + sql "switch ${catalog}" + [false, true].each { scannerV2 -> + sql "set enable_file_scanner_v2 = ${scannerV2}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + // Small batches exercise residual filtering across pages and prohibit unsafe early LIMIT. + sql "set batch_size = 1" + def tag = "dv_${docvalues}_v2_${scannerV2}_parallel_${parallel}" + quickTest("${tag}_source", "select row_id, limited, normalized, nullable_value, body, copied " + + "from doris_source_safety_a order by row_id") + predicates.eachWithIndex { predicate, i -> + def query = "select row_id from doris_source_safety_a where ${predicate} order by row_id" + quickTest("${tag}_predicate_${i}", query, + "select row_id from ${expectedTable} where ${predicate} order by row_id") + quickTest("${tag}_predicate_${i}", query) + } + def selectedValueQuery = "select row_id, multi_body, doconly from doris_source_safety_a order by row_id" + quickTest("${tag}_selected_values", selectedValueQuery, + "select row_id, multi_body, doconly from ${expectedTable} order by row_id") + quickTest("${tag}_selected_values", selectedValueQuery) + mixedNativePredicates.eachWithIndex { pair, i -> + def query = "select row_id from doris_source_safety_a where ${pair[0]} order by row_id" + if (!docvalues && i < 3) { + // Source STRING OR branches cannot share a local residual with esquery. + test { + sql query + exception "Cannot evaluate esquery locally" + } + } else { + quickTest("${tag}_native_${i}", query, + "select row_id from ${expectedTable} where ${pair[1]} order by row_id") + quickTest("${tag}_native_${i}", query) + } + } + ["doris_source_safety_hidden", "doris_source_safety_included"].eachWithIndex { index, i -> + def query = "select row_id from ${index} where tenant = 'B' or ${nativeTerm} order by row_id" + if (!docvalues) { + test { + sql query + exception "Cannot evaluate esquery locally" + } + } else { + quickTest("${tag}_preserved_source_${i}", query, + "select row_id from ${expectedTable} where tenant = 'B' or tenant = 'A' order by row_id") + quickTest("${tag}_preserved_source_${i}", query) + } + } + quickTest("${tag}_arrays", "select row_id, limited, body from doris_source_safety_arrays order by row_id", + "select row_id, limited, body from ${arrayExpectedTable} order by row_id") + quickTest("${tag}_arrays", "select row_id, limited, body from doris_source_safety_arrays order by row_id") + arrayPredicates.eachWithIndex { predicate, i -> + def query = "select row_id from doris_source_safety_arrays where ${predicate} order by row_id" + quickTest("${tag}_array_predicate_${i}", query, + "select row_id from ${arrayExpectedTable} where ${predicate} order by row_id") + quickTest("${tag}_array_predicate_${i}", query) + } + // Projecting limited forces source reads for the entire scan, including sibling. + // Compare its STRING predicates to an internal table, not to ES array membership. + siblingPredicates.eachWithIndex { predicate, i -> + def query = "select row_id, limited, sibling from doris_source_safety_arrays " + + "where ${predicate} order by row_id" + quickTest("${tag}_sibling_${i}", query, + "select row_id, limited, sibling from ${arrayExpectedTable} where ${predicate} order by row_id") + quickTest("${tag}_sibling_${i}", query) + } + def unprojectedSibling = "select row_id, limited from doris_source_safety_arrays " + + "where sibling = '[\"cat\"]' order by row_id" + quickTest("${tag}_sibling_filter_only", unprojectedSibling, + "select row_id, limited from ${arrayExpectedTable} where sibling = '[\"cat\"]' order by row_id") + quickTest("${tag}_sibling_filter_only", unprojectedSibling) + quickTest("${tag}_sibling_limit", "select row_id, limited, sibling from doris_source_safety_arrays " + + "where sibling = 'cat' limit 1") + def onlySibling = "select sibling from doris_source_safety_arrays where row_id = 1 and sibling = 'cat'" + quickTest("${tag}_sibling_docvalue_control", onlySibling, + docvalues ? "select cast('cat' as string)" : "select cast('cat' as string) where false") + quickTest("${tag}_sibling_docvalue_control", onlySibling) + def nativeSibling = """esquery(sibling, '{"term":{"sibling":"cat"}}')""" + def nativeSiblingQuery = "select row_id, limited, sibling from doris_source_safety_arrays " + + "where ${nativeSibling} order by row_id" + quickTest("${tag}_sibling_native", nativeSiblingQuery, + "select row_id, limited, sibling from ${arrayExpectedTable} where row_id in (1,2,3) order by row_id") + quickTest("${tag}_sibling_native", nativeSiblingQuery) + def siblingAnd = "select row_id, limited, sibling from doris_source_safety_arrays " + + "where sibling = 'cat' and ${nativeSibling} order by row_id" + quickTest("${tag}_sibling_native_and", siblingAnd, + "select row_id, limited, sibling from ${arrayExpectedTable} where sibling = 'cat' order by row_id") + quickTest("${tag}_sibling_native_and", siblingAnd) + test { + sql "select limited, sibling from doris_source_safety_arrays where sibling = 'cat' or ${nativeSibling}" + exception "Cannot evaluate esquery locally" + } + // Native esquery intentionally matches array elements; ordinary SQL above compares + // the STRING Doris reads. Keep both contracts explicit instead of guessing scalar shape. + def nativeArrayQuery = "select row_id from doris_source_safety_arrays " + + "where esquery(limited, '{\"term\":{\"limited\":\"cat\"}}') order by row_id" + def nativeArrayIds = request("POST", "doris_source_safety_arrays/_search", + [size: 100, query: [term: [limited: "cat"]]]).hits.hits.collect { it._source.row_id }.sort() + def nativeArrayOracle = nativeArrayIds.collect { + "select cast(${it} as bigint) as row_id" + }.join(" union all ") + quickTest("${tag}_native_arrays", nativeArrayQuery, + "select * from (${nativeArrayOracle}) t order by row_id") + quickTest("${tag}_native_arrays", nativeArrayQuery) + // These combinations need both source evaluation and remote-only ES semantics. + // Reject them while planning rather than dispatching the fake esquery BE function. + ["limited = 'elephant' or ${nativeTerm}", "limited = 'cat' or ${nativeTerm}", + "limited in ('cat', 'four') or ${nativeTerm}", "body = 'cat' or ${nativeTerm}", + "not (limited = 'cat' and ${nativeTerm})", "not (limited = 'elephant' and ${nativeTerm})"].each { predicate -> + test { + sql "select row_id from doris_source_safety_a where ${predicate}" + exception "Cannot evaluate esquery locally" + } + } + ["limited = 'elephant'", "limited is null"].eachWithIndex { predicate, i -> + quickTest("${tag}_limit_${i}", "select row_id from doris_source_safety_a " + + "where ${predicate} limit 1") + } + quickTest("${tag}_hidden", "select row_id, limited from doris_source_safety_hidden " + + "where limited is null order by row_id") + ["doris_source_filtered", "doris_source_multi", "doris_source_routed", + "doris_source_filtered_routed", "doris_source_empty", "doris_source_unfiltered", + "doris_source_with_empty"].each { alias -> + // Compare to ES itself: both Doris scan modes previously shared the same metadata path. + def hits = request("POST", "${alias}/_search", [size: 100, _source: ["row_id"], + query: [match_all: [:]]]).hits.hits + def ids = hits.collect { it._source.row_id }.sort() + def oracle = ids.isEmpty() ? "select cast(null as bigint) as row_id where false" + : ids.collect { "select cast(${it} as bigint) as row_id" }.join(" union all ") + def query = "select row_id from ${alias} order by row_id" + quickTest("${tag}_${alias}", query, "select * from (${oracle}) t order by row_id") + quickTest("${tag}_${alias}", query) + quickTest("${tag}_${alias}_count", "select count(*) from ${alias}") + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set batch_size = ${originalBatch}" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy b/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy new file mode 100644 index 00000000000000..e1b846b9faa61a --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_source_value_boundaries", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + def endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + + def settings = [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [folded: [type: "custom", filter: ["lowercase"]]]]] + request("DELETE", "doris_source_value_boundaries") + request("PUT", "doris_source_value_boundaries", [settings: settings, mappings: [ + _meta: [doris: [array_fields: ["numbers", "truths", "days"]]], properties: [ + row_id: [type: "long"], a: [type: "keyword", normalizer: "folded"], obj: [type: "object"], + n: [type: "long"], f: [type: "float"], d: [type: "double"], i: [type: "integer"], s: [type: "short"], + numbers: [type: "long"], truths: [type: "boolean"], days: [type: "date"]]]]) + // BigDecimal keeps an unquoted floating JSON token rather than an exact integer/string token. + [[row_id: 1, a: "ONE", obj: [[x: 1]], n: new BigDecimal("-9223372036854775808.0"), + f: "", d: "", i: "0x1.8p1", s: "3d", numbers: [null, "", 1, null], + truths: [null, true, false, null], days: [null, "2024-01-01", null]], + [row_id: 2, a: "TWO", obj: [], n: "", f: "0x1.8p1f", d: "3d", i: "+3F", s: " 3d ", + numbers: [null], truths: [null], days: []], + [row_id: 3, a: "THREE", obj: [null], n: null, + f: "1.000000059604644775390625000000000000000000000001", d: 1.5, i: "-0x1.8p1", s: "-3d", + numbers: [], truths: [], days: [null]], + [row_id: 4, a: "FOUR", obj: [x: 1], n: new BigDecimal("9223372036854775808.0"), + f: 2.5, d: 2.5, i: 3, s: 3]].each { row -> + request("PUT", "doris_source_value_boundaries/_doc/${row.row_id}", row) + } + request("POST", "doris_source_value_boundaries/_refresh") + + ["doris_inherited_mapping", "doris_safe_mapping"].eachWithIndex { index, j -> + request("DELETE", index) + def indexSettings = new LinkedHashMap(settings) + indexSettings["index.mapping.ignore_malformed"] = (j == 0) + indexSettings["index.mapping.coerce"] = (j != 0) + request("PUT", index, [settings: indexSettings, mappings: [properties: [ + row_id: [type: "long"], a: [type: "keyword", normalizer: "folded"], b: [type: "long"], + c: [type: "long", ignore_malformed: false]]]]) + request("PUT", "${index}/_doc/1", [row_id: j + 1, a: "VALUE", b: "5", c: 5]) + request("POST", "${index}/_refresh") + } + request("POST", "_aliases", [actions: [ + [add: [index: "doris_inherited_mapping", alias: "doris_mapping_boundaries_alias"]], + [add: [index: "doris_safe_mapping", alias: "doris_mapping_boundaries_alias"]]]]) + + sql "drop table if exists es_source_value_boundaries_expected" + sql """create table es_source_value_boundaries_expected ( + row_id bigint,a string,obj json,n bigint,f float,d double,i int,s smallint, + numbers array,truths array,days array + ) distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql """insert into es_source_value_boundaries_expected values + (1,'ONE','[{"x":1}]',-9223372036854775808,null,null,3,3, + [null,null,1,null],[null,true,false,null],[null,'2024-01-01 00:00:00',null]), + (2,'TWO','[]',null,3,3,3,3,[null],[null],[]), + (3,'THREE','[null]',null,1.00000011920928955078125,1.5,-3,-3,[],[],[null]), + (4,'FOUR','{"x":1}',9223372036854775807,2.5,2.5,3,3,null,null,null)""" + def expected = "internal.${context.dbName}.es_source_value_boundaries_expected" + sql "drop table if exists es_inherited_mapping_expected" + sql """create table es_inherited_mapping_expected (row_id bigint,a string,b bigint,c bigint) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into es_inherited_mapping_expected values (1,'VALUE',5,5),(2,'VALUE',5,5)" + def mappingExpected = "internal.${context.dbName}.es_inherited_mapping_expected" + def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalZone = sql("show variables like 'time_zone'")[0][1] + try { + sql "set time_zone = '+00:00'" + [true, false].each { docvalues -> + def catalog = "es_source_value_boundaries_${docvalues}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties("type"="es","hosts"="${endpoint}", + "nodes_discovery"="false","enable_docvalue_scan"="${docvalues}")""" + sql "switch ${catalog}" + [false, true].each { scanner -> + sql "set enable_file_scanner_v2 = ${scanner}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + def prefix = "dv_${docvalues}_v2_${scanner}_parallel_${parallel}" + ["true", "n is null", "f is null", "i = 3", "n < 0"].eachWithIndex { predicate, j -> + def projection = "row_id,a,cast(obj as string),n,f,d,i,s,numbers,truths,days" + def query = "select ${projection} from doris_source_value_boundaries " + + "where ${predicate} order by row_id" + quickTest("${prefix}_source_${j}", query, + "select ${projection} from ${expected} where ${predicate} order by row_id") + quickTest("${prefix}_source_${j}", query) + } + ["cast(obj as string)", "n", "f", "i"].eachWithIndex { projection, j -> + def query = "select row_id,${projection} from doris_source_value_boundaries order by row_id" + quickTest("${prefix}_control_${j}", query, + "select row_id,${projection} from ${expected} order by row_id") + quickTest("${prefix}_control_${j}", query) + } + ["doris_inherited_mapping", "doris_safe_mapping", "doris_mapping_boundaries_alias"].eachWithIndex { + index, j -> + def rowPredicate = j == 0 ? "row_id=1" : (j == 1 ? "row_id=2" : "true") + ["b is null", "b is not null", "b=5", "c=5"].eachWithIndex { predicate, k -> + def query = "select row_id,a,b,c from ${index} where ${predicate} order by row_id" + quickTest("${prefix}_mapping_${j}_${k}", query, + "select row_id,a,b,c from ${mappingExpected} " + + "where ${rowPredicate} and ${predicate} order by row_id") + quickTest("${prefix}_mapping_${j}_${k}", query) + } + } + } + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalV2}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalZone}'" + } +} From 76a49a6f7377a1395244a75c8239e602c5f78a73 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 14:21:39 +0800 Subject: [PATCH 2/7] [feature](be) Add session controls for Parquet page indexes (#69971) Use the branch-4.2 session-variable annotation and retain the upstream optional Thrift field IDs. (cherry picked from commit 035a843648df12bf814ff946c96f8d1eff09eb87) --- be/src/format/transformer/vparquet_writer.cpp | 17 +- .../transformer/vparquet_writer_test.cpp | 226 ++++++++++++++++++ .../org/apache/doris/qe/SessionVariable.java | 20 ++ .../qe/SessionVariableParquetWriterTest.java | 72 ++++++ gensrc/thrift/PaloInternalService.thrift | 4 + 5 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariableParquetWriterTest.java diff --git a/be/src/format/transformer/vparquet_writer.cpp b/be/src/format/transformer/vparquet_writer.cpp index 7ec66e56b28175..7ec34b7e3b9507 100644 --- a/be/src/format/transformer/vparquet_writer.cpp +++ b/be/src/format/transformer/vparquet_writer.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,7 @@ namespace doris { #include "common/compile_check_begin.h" namespace { +constexpr int64_t HADOOP_LZ4_DEFAULT_BUFFER_SIZE = 256 * 1024; arrow::MemoryPool* get_arrow_memory_pool() { auto* pool = ExecEnv::GetInstance()->arrow_memory_pool(); @@ -152,7 +154,6 @@ void ParquetBuildHelper::build_compression_type( break; } case TParquetCompressionType::LZ4_HADOOP: { - constexpr int64_t HADOOP_LZ4_DEFAULT_BUFFER_SIZE = 256 * 1024; // Hadoop-framed LZ4 -> Parquet thrift codec "LZ4" (deprecated). This matches what // Spark/Iceberg writes for `write.parquet.compression-codec=lz4`. Arrow 17 emits one // Hadoop LZ4 block per Parquet page/dictionary page, while Hadoop JVM readers default to @@ -224,6 +225,20 @@ Status VParquetWriter::_parse_properties() { ::parquet::WriterProperties::Builder builder; ParquetBuildHelper::build_compression_type(builder, _parquet_options.compression_type); ParquetBuildHelper::build_version(builder, _parquet_options.parquet_version); + const auto& query_options = _state->query_options(); + // Preserve the current enabled default explicitly, including when older FEs omit the field. + if (query_options.enable_parquet_write_page_index) { + builder.enable_write_page_index(); + } else { + builder.disable_write_page_index(); + } + int64_t page_size = query_options.parquet_write_data_page_size; + DORIS_CHECK_GT(page_size, 0); + // A 1 MiB session target must still respect the 128 KiB Hadoop-framed LZ4 limit. + if (_parquet_options.compression_type == TParquetCompressionType::LZ4_HADOOP) { + page_size = std::min(page_size, HADOOP_LZ4_DEFAULT_BUFFER_SIZE / 2); + } + builder.data_pagesize(page_size); if (_parquet_options.parquet_disable_dictionary) { builder.disable_dictionary(); } else { diff --git a/be/test/format/transformer/vparquet_writer_test.cpp b/be/test/format/transformer/vparquet_writer_test.cpp index 3877f42221a8a1..929ebf2d09cf66 100644 --- a/be/test/format/transformer/vparquet_writer_test.cpp +++ b/be/test/format/transformer/vparquet_writer_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -112,10 +113,235 @@ class VParquetWriterTest : public testing::Test { EXPECT_EQ(offsets, repeated); } + Block page_index_block() { + auto type = std::make_shared(); + auto nullable = make_nullable(type); + auto ids = type->create_column(); + auto values = nullable->create_column(); + auto nulls = nullable->create_column(); + for (int32_t row = 0; row < 12000; ++row) { + ids->insert(Field::create_field(row)); + if (row % 7 == 0) { + values->insert_default(); + } else { + values->insert(Field::create_field(row % 64)); + } + nulls->insert_default(); + } + return Block({{std::move(ids), type, "id"}, + {std::move(values), nullable, "value"}, + {std::move(nulls), nullable, "all_null"}}); + } + + void check_page_indexes(bool enabled) { + auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + ASSERT_EQ(reader->metadata()->num_row_groups(), 1); + auto group = reader->metadata()->RowGroup(0); + ASSERT_EQ(group->num_rows(), 12000); + for (int column = 0; column < group->num_columns(); ++column) { + auto chunk = group->ColumnChunk(column); + EXPECT_EQ(chunk->GetColumnIndexLocation().has_value(), enabled); + EXPECT_EQ(chunk->GetOffsetIndexLocation().has_value(), enabled); + if (!enabled) { + continue; + } + auto index_reader = reader->GetPageIndexReader(); + ASSERT_NE(index_reader, nullptr); + auto indexes = index_reader->RowGroup(0); + ASSERT_NE(indexes, nullptr); + auto column_index = indexes->GetColumnIndex(column); + auto offset_index = indexes->GetOffsetIndex(column); + ASSERT_NE(column_index, nullptr); + ASSERT_NE(offset_index, nullptr); + const auto& pages = offset_index->page_locations(); + ASSERT_FALSE(pages.empty()); + EXPECT_EQ(pages.size(), column_index->null_pages().size()); + EXPECT_EQ(pages.front().offset, chunk->data_page_offset()); + EXPECT_EQ(pages.front().first_row_index, 0); + const auto start = chunk->has_dictionary_page() ? chunk->dictionary_page_offset() + : chunk->data_page_offset(); + int64_t previous_row = -1; + int64_t previous_end = start; + for (const auto& page : pages) { + EXPECT_GT(page.first_row_index, previous_row); + EXPECT_LT(page.first_row_index, group->num_rows()); + EXPECT_GE(page.offset, previous_end); + EXPECT_GT(page.compressed_page_size, 0); + EXPECT_LE(page.offset + page.compressed_page_size, + start + chunk->total_compressed_size()); + previous_row = page.first_row_index; + previous_end = page.offset + page.compressed_page_size; + } + if (column == 0) { + EXPECT_GT(pages.size(), 1); + } else if (column == 2) { + EXPECT_TRUE(std::all_of(column_index->null_pages().begin(), + column_index->null_pages().end(), + [](bool value) { return value; })); + } + } + auto input = arrow::io::ReadableFile::Open(_file_path); + ASSERT_TRUE(input.ok()) << input.status(); + auto arrow_reader = ::parquet::arrow::OpenFile(*input, arrow::default_memory_pool()); + ASSERT_TRUE(arrow_reader.ok()) << arrow_reader.status(); + auto table = (*arrow_reader)->ReadTable(); + ASSERT_TRUE(table.ok()) << table.status(); + EXPECT_EQ((*table)->num_rows(), 12000); + EXPECT_EQ((*table)->column(1)->null_count(), (12000 + 6) / 7); + EXPECT_EQ((*table)->column(2)->null_count(), 12000); + const auto& ids = assert_cast(*(*table)->column(0)->chunk(0)); + EXPECT_EQ(ids.Value(0), 0); + EXPECT_EQ(ids.Value(11999), 11999); + } + std::string _file_path; std::shared_ptr _fs; }; +TEST_F(VParquetWriterTest, ParquetPageIndexToggle) { + for (const bool enabled : {false, true}) { + for (const bool dictionary : {false, true}) { + for (const auto version : + {TParquetVersion::PARQUET_1_0, TParquetVersion::PARQUET_2_LATEST}) { + TQueryOptions query_options; + query_options.__set_enable_parquet_write_page_index(enabled); + query_options.__set_parquet_write_data_page_size(1024); + RuntimeState state(query_options, TQueryGlobals()); + state.set_timezone("UTC"); + auto block = page_index_block(); + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); + io::FileWriterPtr file; + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); + ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY, + .parquet_version = version, + .parquet_disable_dictionary = !dictionary}; + VParquetWriter writer(&state, file.get(), exprs, + std::vector {"id", "value", "all_null"}, false, + options); + ASSERT_TRUE(writer.open().ok()); + ASSERT_TRUE(writer.write(block).ok()); + ASSERT_TRUE(writer.close().ok()); + check_page_indexes(enabled); + } + } + } +} + +TEST_F(VParquetWriterTest, ParquetPageIndexDataPageVersions) { + for (const auto page_version : + {::parquet::ParquetDataPageVersion::V1, ::parquet::ParquetDataPageVersion::V2}) { + TQueryOptions query_options; + query_options.__set_enable_parquet_write_page_index(true); + query_options.__set_parquet_write_data_page_size(1024); + RuntimeState state(query_options, TQueryGlobals()); + auto block = page_index_block(); + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); + io::FileWriterPtr file; + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); + const std::vector names {"id", "value", "all_null"}; + ParquetFileOptions options {.compression_type = TParquetCompressionType::SNAPPY, + .parquet_version = TParquetVersion::PARQUET_2_LATEST}; + VParquetWriter writer(&state, file.get(), exprs, names, false, options); + ASSERT_TRUE(writer._parse_properties().ok()); + // Exercise both page layouts with the production writer properties; this does not add + // a SQL option for changing the existing data-page version. + auto properties = ::parquet::WriterProperties::Builder(*writer._parquet_writer_properties) + .data_page_version(page_version) + ->build(); + auto converter = writer._create_arrow_block_convertor(block.get_data_types(), names, "UTC", + cctz::utc_time_zone(), false); + ASSERT_TRUE(converter->init().ok()); + std::shared_ptr batch; + ASSERT_TRUE(converter->convert_to_arrow(block, arrow::default_memory_pool(), &batch).ok()); + auto result = ::parquet::arrow::FileWriter::Open( + *batch->schema(), arrow::default_memory_pool(), writer._outstream, properties, + writer._arrow_properties); + ASSERT_TRUE(result.ok()) << result.status(); + auto arrow_writer = std::move(result).ValueOrDie(); + ASSERT_TRUE(arrow_writer->WriteRecordBatch(*batch).ok()); + ASSERT_TRUE(arrow_writer->Close().ok()); + ASSERT_TRUE(writer._outstream->Close().ok()); + check_page_indexes(true); + } +} + +TEST_F(VParquetWriterTest, ParquetPageSizePreservesHadoopLz4Limit) { + for (const auto codec : {TParquetCompressionType::LZ4_HADOOP, TParquetCompressionType::LZ4}) { + for (const int32_t requested_size : {65536, 262144}) { + TQueryOptions query_options; + query_options.__set_parquet_write_data_page_size(requested_size); + RuntimeState state(query_options, TQueryGlobals()); + state.set_timezone("UTC"); + auto block = page_index_block(); + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); + io::FileWriterPtr file; + ASSERT_TRUE(_fs->create_file(_file_path, &file).ok()); + ParquetFileOptions options {.compression_type = codec, + .parquet_version = TParquetVersion::PARQUET_1_0}; + VParquetWriter writer(&state, file.get(), exprs, + std::vector {"id", "value", "all_null"}, false, + options); + ASSERT_TRUE(writer.open().ok()); + EXPECT_TRUE(writer._parquet_writer_properties->page_index_enabled()); + EXPECT_EQ(writer._parquet_writer_properties->data_pagesize(), + codec == TParquetCompressionType::LZ4_HADOOP + ? std::min(requested_size, 131072) + : requested_size); + ASSERT_TRUE(writer.close().ok()); + } + } +} + +TEST_F(VParquetWriterTest, HiveAndIcebergParquetPageIndexOptions) { + const std::string json = + R"({"type":"struct","fields":[{"id":1,"name":"id","required":true,"type":"int"},{"id":2,"name":"value","required":false,"type":"int"},{"id":3,"name":"all_null","required":false,"type":"int"}]})"; + auto schema = iceberg::SchemaParser::from_json(json); + const std::map hadoop_conf; + for (const bool iceberg_writer : {false, true}) { + for (const bool enabled : {false, true}) { + TQueryOptions query_options; + query_options.__set_enable_parquet_write_page_index(enabled); + query_options.__set_parquet_write_data_page_size(1024); + RuntimeState state(query_options, TQueryGlobals()); + state.set_timezone("UTC"); + auto block = page_index_block(); + auto exprs = MockSlotRef::create_mock_contexts(block.get_data_types()); + const auto name = "page_index_" + UniqueId::gen_uid().to_string(); + _file_path = "./" + name + "-0.parquet"; + TDataSink sink; + if (iceberg_writer) { + VIcebergPartitionWriter writer(sink, {}, exprs, *schema, &json, + {"id", "value", "all_null"}, + {.write_path = ".", + .original_write_path = ".", + .target_path = ".", + .file_type = TFileType::FILE_LOCAL, + .broker_addresses = {}}, + name, 0, TFileFormatType::FORMAT_PARQUET, + TFileCompressType::PLAIN, hadoop_conf); + ASSERT_TRUE(writer.open(&state, nullptr, nullptr).ok()); + ASSERT_TRUE(writer.write(block).ok()); + ASSERT_TRUE(writer.close(Status::OK()).ok()); + } else { + VHivePartitionWriter writer(sink, "", TUpdateMode::APPEND, exprs, + {"id", "value", "all_null"}, + {.write_path = ".", + .original_write_path = ".", + .target_path = ".", + .file_type = TFileType::FILE_LOCAL, + .broker_addresses = {}}, + name, 0, TFileFormatType::FORMAT_PARQUET, + TFileCompressType::PLAIN, nullptr, hadoop_conf); + ASSERT_TRUE(writer.open(&state, nullptr).ok()); + ASSERT_TRUE(writer.write(block).ok()); + ASSERT_TRUE(writer.close(Status::OK()).ok()); + } + check_page_indexes(enabled); + ASSERT_TRUE(_fs->delete_file(_file_path).ok()); + } + } +} + TEST_F(VParquetWriterTest, WritesIcebergVariantAndCollectsLogicalMetrics) { auto variant_type = std::make_shared(); auto nullable_variant_type = make_nullable(variant_type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a5f6d3891ff33f..0c21e481a863bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -567,6 +567,8 @@ public String toString() { public static final String ENABLE_PAGE_CACHE = "enable_page_cache"; public static final String ENABLE_PARQUET_FILE_PAGE_CACHE = "enable_parquet_file_page_cache"; + public static final String ENABLE_PARQUET_WRITE_PAGE_INDEX = "enable_parquet_write_page_index"; + public static final String PARQUET_WRITE_DATA_PAGE_SIZE = "parquet_write_data_page_size"; public static final String MINIDUMP_PATH = "minidump_path"; @@ -2637,6 +2639,21 @@ public Map getForceEagerAggHintMap() { needForward = true) public boolean enableParquetFilePageCache = true; + @VariableMgr.VarAttr(name = ENABLE_PARQUET_WRITE_PAGE_INDEX, needForward = true, + description = {"控制新写入的 Parquet 文件是否包含 ColumnIndex 和 OffsetIndex 元数据。默认值为 true。", + "Write ColumnIndex and OffsetIndex metadata to new Parquet files. Default is true."}) + public boolean enableParquetWritePageIndex = true; + + @VariableMgr.VarAttr(name = PARQUET_WRITE_DATA_PAGE_SIZE, needForward = true, + checker = "checkParquetWriteDataPageSize", + description = {"Parquet 未压缩数据页的目标字节数,必须为正整数,默认值为 1048576。", + "Target uncompressed Parquet data-page size in bytes. Must be positive; default is 1048576."}) + public int parquetWriteDataPageSize = 1048576; + + public void checkParquetWriteDataPageSize(String value) throws Exception { + checkFieldValue(PARQUET_WRITE_DATA_PAGE_SIZE, 1, value); + } + @VariableMgr.VarAttr(name = ENABLE_FOLD_NONDETERMINISTIC_FN) public boolean enableFoldNondeterministicFn = false; @@ -5941,6 +5958,9 @@ public TQueryOptions toThrift() { } tResult.setEnableParquetLazyMat(enableParquetLazyMat); + // Query options carry the same write policy to OUTFILE and catalog sinks. + tResult.setEnableParquetWritePageIndex(enableParquetWritePageIndex); + tResult.setParquetWriteDataPageSize(parquetWriteDataPageSize); tResult.setEnableOrcLazyMat(enableOrcLazyMat); tResult.setEnableParquetFilterByMinMax(enableParquetFilterByMinMax); tResult.setEnableParquetFilterByBloomFilter(enableParquetFilterByBloomFilter); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariableParquetWriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariableParquetWriterTest.java new file mode 100644 index 00000000000000..f1bcf5977513e3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariableParquetWriterTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.SetType; +import org.apache.doris.analysis.SetVar; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.thrift.TQueryOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class SessionVariableParquetWriterTest { + private static final String WRITE_INDEX = "enable_parquet_write_page_index"; + private static final String PAGE_SIZE = "parquet_write_data_page_size"; + + @Test + void testDefaults() { + TQueryOptions options = new SessionVariable().toThrift(); + Assertions.assertTrue(options.isEnableParquetWritePageIndex()); + Assertions.assertEquals(1048576, options.getParquetWriteDataPageSize()); + } + + @Test + void testForwardingAndThriftSerialization() throws Exception { + SessionVariable session = new SessionVariable(); + VariableMgr.setVar(session, new SetVar(SetType.SESSION, WRITE_INDEX, new StringLiteral("false"))); + VariableMgr.setVar(session, new SetVar(SetType.SESSION, PAGE_SIZE, new StringLiteral("262144"))); + SessionVariable forwarded = new SessionVariable(); + forwarded.setForwardedSessionVariables(session.getForwardVariables()); + TQueryOptions options = forwarded.toThrift(); + Assertions.assertTrue(options.isSetEnableParquetWritePageIndex()); + Assertions.assertFalse(options.isEnableParquetWritePageIndex()); + Assertions.assertTrue(options.isSetParquetWriteDataPageSize()); + Assertions.assertEquals(262144, options.getParquetWriteDataPageSize()); + } + + @ParameterizedTest + @ValueSource(strings = {"1", "1048576", "2147483647"}) + void testValidPageSize(String value) throws Exception { + SessionVariable session = new SessionVariable(); + VariableMgr.setVar(session, new SetVar(SetType.SESSION, PAGE_SIZE, new StringLiteral(value))); + Assertions.assertEquals(Integer.parseInt(value), session.toThrift().getParquetWriteDataPageSize()); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "-1", "2147483648", "invalid"}) + void testInvalidPageSize(String value) { + SessionVariable session = new SessionVariable(); + Assertions.assertThrows(DdlException.class, () -> VariableMgr.setVar(session, + new SetVar(SetType.SESSION, PAGE_SIZE, new StringLiteral(value)))); + Assertions.assertEquals(1048576, session.toThrift().getParquetWriteDataPageSize()); + } +} diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 2ad5c12397b0c0..e8a529b8795752 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -518,6 +518,10 @@ struct TQueryOptions { // Push LIMIT into SegmentIterator when safe. 231: optional bool enable_segment_limit_pushdown = true + + // Optional Parquet output indexes; absent fields retain the existing writer defaults. + 238: optional bool enable_parquet_write_page_index = true; + 239: optional i32 parquet_write_data_page_size = 1048576; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. From 5617df0161f2779b0e62f8a11e26ab140bae6a7f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 14:23:03 +0800 Subject: [PATCH 3/7] [fix](regression) Repair ES source predicate regression cases (#70155) Preserve branch-specific ES coverage and adapt predicate-plan assertions to the branch-4.2 EXPLAIN fields. (cherry picked from commit ec7f806fa3907f9be936cbaa789eb0a20476d41c) --- .../external_table_p2/es/test_es_query.out | 24 +-- .../es/test_es_query_nereids.out | 2 + .../test_es_query_predicate_correctness.out | 63 +++++++ .../es/test_es_epoch_millis.groovy | 3 +- .../es/test_es_mapping_boundaries.groovy | 3 +- .../test_es_numeric_source_semantics.groovy | 3 +- .../external_table_p2/es/test_es_query.groovy | 29 +++ .../es/test_es_query_nereids.groovy | 18 +- ...test_es_query_predicate_correctness.groovy | 166 ++++++++++++++++++ .../es/test_es_scalar_materialization.groovy | 3 +- .../es/test_es_source_fallback_types.groovy | 27 ++- .../es/test_es_source_semantics.groovy | 3 +- .../es/test_es_source_value_boundaries.groovy | 3 +- 13 files changed, 318 insertions(+), 29 deletions(-) create mode 100644 regression-test/data/external_table_p2/es/test_es_query_predicate_correctness.out create mode 100644 regression-test/suites/external_table_p2/es/test_es_query_predicate_correctness.groovy diff --git a/regression-test/data/external_table_p2/es/test_es_query.out b/regression-test/data/external_table_p2/es/test_es_query.out index e13f30c3e3f8a7..e2cfc0d7da53c8 100644 --- a/regression-test/data/external_table_p2/es/test_es_query.out +++ b/regression-test/data/external_table_p2/es/test_es_query.out @@ -235,6 +235,7 @@ value1 value2 string1 text#1 string2 text2 string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_5_30 -- string1 text#1 @@ -389,6 +390,7 @@ value1 value2 string1 text#1 string2 text2 string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_6_30 -- string1 text#1 @@ -500,10 +502,6 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_7_23 -- -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string_ignore_above_10 2022-08-08T20:10:10 text_ignore_above_10 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_7_24 -- @@ -583,6 +581,7 @@ string1 text#1 string2 text2 string3 text3_4*5 string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_7_36 -- string1 text#1 @@ -704,10 +703,6 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_8_21 -- -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string_ignore_above_10 2022-08-08T20:10:10 text_ignore_above_10 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_8_22 -- @@ -787,6 +782,7 @@ string1 text#1 string2 text2 string3 text3_4*5 string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_8_34 -- string1 text#1 @@ -1048,6 +1044,7 @@ value1 value2 string1 text#1 string2 text2 string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_5_30 -- string1 text#1 @@ -1202,6 +1199,7 @@ value1 value2 string1 text#1 string2 text2 string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_6_30 -- string1 text#1 @@ -1313,10 +1311,6 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_7_23 -- -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string_ignore_above_10 2022-08-08T20:10:10 text_ignore_above_10 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_7_24 -- @@ -1396,6 +1390,7 @@ string1 text#1 string2 text2 string3 text3_4*5 string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_7_36 -- string1 text#1 @@ -1517,10 +1512,6 @@ I'm not null or empty [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_8_21 -- -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string4 2022-08-08T20:10:10 text3_4*5 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N \N string_ignore_above_10 2022-08-08T20:10:10 text_ignore_above_10 6 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 4444 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N \N string2 2022-08-08T12:10:10 text2 4 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 2222 -[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N I'm not null or empty \N string3 2022-08-09T00:40:10 text3_4*5 5 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 3333 [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 -- !sql_8_22 -- @@ -1600,6 +1591,7 @@ string1 text#1 string2 text2 string3 text3_4*5 string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 -- !sql_8_34 -- string1 text#1 diff --git a/regression-test/data/external_table_p2/es/test_es_query_nereids.out b/regression-test/data/external_table_p2/es/test_es_query_nereids.out index 0c9bed926df685..0fb98abc0dd491 100644 --- a/regression-test/data/external_table_p2/es/test_es_query_nereids.out +++ b/regression-test/data/external_table_p2/es/test_es_query_nereids.out @@ -7,6 +7,8 @@ string1 text#1 3.14 2022-08-08T00:00 string2 text2 4.0 2022-08-08T00:00 string3 text3_4*5 5.0 2022-08-08T00:00 +-- !source_date_mismatch -- + -- !sql64 -- string2 text2 4.0 2022-08-08T00:00 diff --git a/regression-test/data/external_table_p2/es/test_es_query_predicate_correctness.out b/regression-test/data/external_table_p2/es/test_es_query_predicate_correctness.out new file mode 100644 index 00000000000000..b768446ee2de39 --- /dev/null +++ b/regression-test/data/external_table_p2/es/test_es_query_predicate_correctness.out @@ -0,0 +1,63 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !es7_regexp_charclass -- +string1 +string2 + +-- !es7_regexp_dotstar -- +string1 +string2 +string3 +string4 +string_ignore_above_10 + +-- !es7_regexp_alternation -- +string1 +string3 + +-- !es7_like -- +string1 +string2 +string3 +string4 +string_ignore_above_10 + +-- !es7_is_null -- +3 + +-- !es7_is_not_null -- +2 + +-- !es7_total_count -- +5 + +-- !es8_regexp_charclass -- +string1 +string2 + +-- !es8_regexp_dotstar -- +string1 +string2 +string3 +string4 +string_ignore_above_10 + +-- !es8_regexp_alternation -- +string1 +string3 + +-- !es8_like -- +string1 +string2 +string3 +string4 +string_ignore_above_10 + +-- !es8_is_null -- +3 + +-- !es8_is_not_null -- +2 + +-- !es8_total_count -- +5 + diff --git a/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy b/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy index 49dc1561eb1fe6..eecd59c3dbf148 100644 --- a/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_epoch_millis.groovy @@ -72,7 +72,8 @@ suite("test_es_epoch_millis", "p2,external") { (9,'VALUE','1969-12-31 23:59:58'),(10,'VALUE','2023-03-04 02:08:48'), (11,'VALUE',null),(12,'VALUE','1970-01-01 00:00:00'),(13,'VALUE',null),(14,'VALUE',null)""" def expected = "internal.${context.dbName}.es_epoch_millis_expected" - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] def originalZone = sql("show variables like 'time_zone'")[0][1] try { diff --git a/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy b/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy index d3ff3bb8e269b7..72fa2e1a3b4e4a 100644 --- a/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_mapping_boundaries.groovy @@ -95,7 +95,8 @@ suite("test_es_mapping_boundaries", "p2,external") { "amount > 1 or ${nativeTerm}", "flag = true or ${nativeTerm}", "not (flag = true or ${nativeTerm})", "event_time = 1700000000000 or ${nativeTerm}", "not (event_time = 1700000000000 or ${nativeTerm})", "event_time > 1700000000000 or ${nativeTerm}"] - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] try { [true, false].each { docvalues -> diff --git a/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy index 5571132290d275..fdb8dabd59835a 100644 --- a/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy @@ -75,7 +75,8 @@ suite("test_es_numeric_source_semantics", "p2,external") { "c = cast(2.34 as float)", "c = cast(2.33984375 as float)", "c > cast(2.33984375 as float)", "c in (cast(2.34 as float), cast(-2.34 as float))"] def nativeTerm = """esquery(b, '{"term":{"b":2.25}}')""" - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] try { ["enabled", "disabled", "limited"].each { mode -> diff --git a/regression-test/suites/external_table_p2/es/test_es_query.groovy b/regression-test/suites/external_table_p2/es/test_es_query.groovy index eb4baad90fea11..dbfba45b31dd1a 100644 --- a/regression-test/suites/external_table_p2/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query.groovy @@ -193,6 +193,31 @@ suite("test_es_query", "p2,external") { } } + def verifySourcePredicates = { String version -> + // Unfiltered source materialization keeps values omitted by ignore_above out of the oracle's index path. + def expected = "internal.${context.dbName}.es_query_source_expected" + sql "drop table if exists ${expected}" + sql """create table ${expected} (row_id int, test1 string, test2 string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into ${expected} select 0, test1, test2 from test1" + ["test1 like 'string%'", "test2 like 'text%'"].eachWithIndex { predicate, i -> + quickTest("es_${version}_source_like_${i}", + "select test1,test2 from test1 where ${predicate} order by test1,test2", + "select test1,test2 from ${expected} where ${predicate} order by test1,test2") + } + if (version == "7" || version == "8") { + // An absent source field is NULL even when another document has an indexed value. + def levelExpected = "internal.${context.dbName}.es_query_level_expected" + sql "drop table if exists ${levelExpected}" + sql """create table ${levelExpected} (row_id int,test1 string,level string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into ${levelExpected} select 0,test1,level from test1" + quickTest("es_${version}_source_level", + "select test1,level from test1 where level = 'debug' order by test1", + "select test1,level from ${levelExpected} where level = 'debug' order by test1") + } + } + def query_catalogs = { -> sql """switch internal""" // test_v1/test_v2 live in this suite's own database, which the @@ -222,6 +247,7 @@ suite("test_es_query", "p2,external") { order_qt_sql26 """select test9 from test_v2;""" sql """switch test_es_query_es5""" + verifySourcePredicates("5") order_qt_sql_5_02 """select * from test1 where test2='text#1'""" order_qt_sql_5_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql_5_04 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" @@ -255,6 +281,7 @@ suite("test_es_query", "p2,external") { order_qt_sql_5_32 """select test7 from test2;""" sql """switch test_es_query_es6""" + verifySourcePredicates("6") // order_qt_sql_6_01 """show tables""" order_qt_sql_6_02 """select * from test1 where test2='text#1'""" order_qt_sql_6_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" @@ -309,6 +336,7 @@ suite("test_es_query", "p2,external") { assertTrue(containHide) sql """switch test_es_query_es7""" + verifySourcePredicates("7") // order_qt_sql_7_01 """show tables""" order_qt_sql_7_02 """select * from test1 where test2='text#1'""" order_qt_sql_7_03 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" @@ -372,6 +400,7 @@ suite("test_es_query", "p2,external") { order_qt_sql_7_50 """select * from test3_20231005""" sql """switch test_es_query_es8""" + verifySourcePredicates("8") order_qt_sql_8_01 """select * from test1 where test2='text#1'""" order_qt_sql_8_02 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql_8_03 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], json_extract(c_person, '\$.[0]') from test1""" diff --git a/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy index 230c05e7784053..bc0d41f97cae79 100644 --- a/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy @@ -37,7 +37,8 @@ suite("test_es_query_nereids", "p2,external") { "type"="es", "elasticsearch.hosts"="http://${externalEnvIp}:$es_6_port", "elasticsearch.nodes_discovery"="false", - "elasticsearch.keyword_sniff"="true" + "elasticsearch.keyword_sniff"="true", + "enable_docvalue_scan"="false" ); """ @@ -141,7 +142,20 @@ suite("test_es_query_nereids", "p2,external") { sql """switch es6_nereids""" order_qt_sql62 """select test1, test2, test3, test4 from test1 where test2='text#1'""" - order_qt_sql63 """select test1, test2, test3, test4 from test2_20220808 where test4='2022-08-08 08:00:00'""" + // Date-only source strings materialize at local midnight; an indexed UTC match cannot replace this predicate. + def expected = "internal.${context.dbName}.es_nereids_source_expected" + sql "drop table if exists ${expected}" + sql """create table ${expected} (row_id int,test1 string,test2 string,test3 double,test4 datetimev2(0)) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into ${expected} select 0,test1,test2,test3,test4 from test2_20220808" + ["2022-08-08 00:00:00", "2022-08-08 08:00:00"].eachWithIndex { date, i -> + quickTest("source_date_${i}", + "select test1,test2,test3,test4 from test2_20220808 where test4='${date}' order by test1", + "select test1,test2,test3,test4 from ${expected} where test4='${date}' order by test1") + } + order_qt_sql63 """select test1, test2, test3, test4 from test2_20220808 where test4='2022-08-08 00:00:00'""" + order_qt_source_date_mismatch """select test1,test2,test3,test4 from test2_20220808 + where test4='2022-08-08 08:00:00'""" order_qt_sql64 """select test1, test2, test3, test4 from test2_20220808 where substring(test2, 2) = 'ext2'""" // TODO(ftw): should open these annotation when nereids support ARRAY // order_qt_sql62 """select * from test1 where test2='text#1'""" diff --git a/regression-test/suites/external_table_p2/es/test_es_query_predicate_correctness.groovy b/regression-test/suites/external_table_p2/es/test_es_query_predicate_correctness.groovy new file mode 100644 index 00000000000000..6be4e782ba33ab --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_query_predicate_correctness.groovy @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Check SQL predicate results independently of whether ES index semantics permit pushdown. + +suite("test_es_query_predicate_correctness", "p2,external") { + String enabled = context.config.otherConfigs.get("enableEsTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String es_7_port = context.config.otherConfigs.get("es_7_port") + String es_8_port = context.config.otherConfigs.get("es_8_port") + + sql """drop catalog if exists es7_pred_test;""" + sql """drop catalog if exists es8_pred_test;""" + + sql """create catalog if not exists es7_pred_test properties( + "type"="es", + "hosts"="http://${externalEnvIp}:$es_7_port", + "nodes_discovery"="false", + "enable_keyword_sniff"="true" + );""" + + sql """create catalog if not exists es8_pred_test properties( + "type"="es", + "hosts"="http://${externalEnvIp}:$es_8_port", + "nodes_discovery"="false", + "enable_keyword_sniff"="true" + );""" + + def verifySourcePredicates = { String version -> + // SQL predicates apply to materialized source values even when ES omits their keyword index values. + def expected = "internal.${context.dbName}.es_predicate_source_expected" + sql "drop table if exists ${expected}" + sql """create table ${expected} (row_id int, test1 string, message string) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + sql "insert into ${expected} select 0,test1,message from test1" + ["test1 regexp 'string[12]'", "test1 regexp 'string.*'", + "test1 regexp 'string1|string3'", "test1 like 'string%'", + "message is null", "message is not null"].eachWithIndex { predicate, i -> + quickTest("es_${version}_source_predicate_${i}", + "select test1 from test1 where ${predicate} order by test1", + "select test1 from ${expected} where ${predicate} order by test1") + } + } + + // ====================================================================== + // ES 7 tests + // ====================================================================== + sql """switch es7_pred_test""" + verifySourcePredicates("7") + + // Character classes and alternation must retain SQL regular-expression semantics. + order_qt_es7_regexp_charclass """select test1 from test1 where test1 regexp 'string[12]' order by test1""" + + // REGEXP with dot-star + order_qt_es7_regexp_dotstar """select test1 from test1 where test1 regexp 'string.*' order by test1""" + + // REGEXP with alternation + order_qt_es7_regexp_alternation """select test1 from test1 where test1 regexp 'string1|string3' order by test1""" + + // LIKE must include matching source values omitted by ignore_above. + order_qt_es7_like """select test1 from test1 where test1 like 'string%' order by test1""" + + // NULL predicates must agree with the source values used by the local oracle. + order_qt_es7_is_null """select count(*) from test1 where message is null""" + order_qt_es7_is_not_null """select count(*) from test1 where message is not null""" + order_qt_es7_total_count """select count(*) from test1""" + + // ====================================================================== + // ES 8 tests (same queries, verifying both versions) + // ====================================================================== + sql """switch es8_pred_test""" + verifySourcePredicates("8") + + order_qt_es8_regexp_charclass """select test1 from test1 where test1 regexp 'string[12]' order by test1""" + order_qt_es8_regexp_dotstar """select test1 from test1 where test1 regexp 'string.*' order by test1""" + order_qt_es8_regexp_alternation """select test1 from test1 where test1 regexp 'string1|string3' order by test1""" + order_qt_es8_like """select test1 from test1 where test1 like 'string%' order by test1""" + order_qt_es8_is_null """select count(*) from test1 where message is null""" + order_qt_es8_is_not_null """select count(*) from test1 where message is not null""" + order_qt_es8_total_count """select count(*) from test1""" + + // ====================================================================== + // LIMIT and predicate-residual checks (ES 7). + // The 4.2 planner exposes LOCAL_PREDICATES rather than ES terminate_after. + // A residual must keep LIMIT above local filtering. + // ====================================================================== + sql """switch es7_pred_test""" + + // A scan without WHERE has no local predicate. + explain { + sql("select * from test1 limit 5") + notContains "LOCAL_PREDICATES" + } + + // A raw keyword with ignore_above needs a residual, so it cannot terminate the ES scan early. + explain { + sql("select * from test1 where test1 = 'string1' limit 3") + contains "LOCAL_PREDICATES" + } + + // An unchanged numeric docvalue remains eligible for predicate and LIMIT pushdown. + explain { + sql("select test3 from test1 where test3 = 3.14 limit 3") + notContains "LOCAL_PREDICATES" + } + + // A scan without WHERE still has no local predicate. + explain { + sql("select * from test1") + notContains "LOCAL_PREDICATES" + } + + // Case 4: Verify data correctness with LIMIT + // The result count should be <= the limit value + def limitResult = sql """select count(*) from (select * from test1 limit 5) t""" + assertTrue(limitResult[0][0] as int <= 5, "LIMIT 5 should return at most 5 rows") + + // ====================================================================== + // LIMIT and predicate-residual checks (ES 8) + // ====================================================================== + sql """switch es8_pred_test""" + + explain { + sql("select * from test1 limit 5") + notContains "LOCAL_PREDICATES" + } + + explain { + sql("select * from test1 where test1 = 'string1' limit 3") + contains "LOCAL_PREDICATES" + } + + // An unchanged numeric docvalue remains eligible for predicate and LIMIT pushdown. + explain { + sql("select test3 from test1 where test3 = 3.14 limit 3") + notContains "LOCAL_PREDICATES" + } + + explain { + sql("select * from test1") + notContains "LOCAL_PREDICATES" + } + + def limitResult8 = sql """select count(*) from (select * from test1 limit 5) t""" + assertTrue(limitResult8[0][0] as int <= 5, "LIMIT 5 should return at most 5 rows") + + // cleanup + sql """drop catalog if exists es7_pred_test;""" + sql """drop catalog if exists es8_pred_test;""" + } +} diff --git a/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy b/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy index a0616b5b6d619b..bb0363c352b454 100644 --- a/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_scalar_materialization.groovy @@ -100,7 +100,8 @@ suite("test_es_scalar_materialization", "p2,external") { distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" sql "insert into es_retained_range_expected values (1,1,'miss'),(2,2,'miss'),(3,3,'miss'),(4,null,'miss'),(5,1,'hit')" def rangeExpected = "internal.${context.dbName}.es_retained_range_expected" - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] def originalZone = sql("show variables like 'time_zone'")[0][1] def originalRules = sql("show variables like 'disable_nereids_expression_rules'")[0][1] diff --git a/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy b/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy index 034bbf6a995f2d..d650ce858a5306 100644 --- a/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_source_fallback_types.groovy @@ -26,7 +26,7 @@ suite("test_es_source_fallback_types", "p2,external") { String port = context.config.otherConfigs.get("es_8_port") assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") def endpoint = "http://${host}:${port}" - def request = { String method, String path, Object body = null -> + def request = { String method, String path, Object body = null, boolean probeSourceMode = false -> def connection = new URL("${endpoint}/${path}").openConnection() connection.requestMethod = method connection.connectTimeout = 10000 @@ -42,8 +42,18 @@ suite("test_es_source_fallback_types", "p2,external") { return null } def text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + def response = new JsonSlurper().parseText(text) + // Older ES8 images reject this mapping syntax; every other HTTP failure remains fatal. + if (probeSourceMode && status == 400 + && response.error?.type == "mapper_parsing_exception" + && response.error?.caused_by?.reason == + "unknown parameter [mode] on mapper [_source] of type [null]") { + logger.info("The ES fixture does not support _source.mode; its syntax is not covered here, " + + "using enabled:false") + return null + } assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") - return new JsonSlurper().parseText(text) + return response } finally { connection.disconnect() } @@ -70,9 +80,15 @@ suite("test_es_source_fallback_types", "p2,external") { ["flag", "mode"].each { kind -> def disabled = "doris_source_disabled_${kind}" request("DELETE", disabled) - request("PUT", disabled, [settings: settings, mappings: [ + def created = request("PUT", disabled, [settings: settings, mappings: [ _source: kind == "flag" ? [enabled: false] : [mode: "disabled"], - properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]]) + properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]], kind == "mode") + if (created == null) { + // Both syntaxes disable the source. Keep the same row/alias checks on baseline ES8. + request("PUT", disabled, [settings: settings, mappings: [ + _source: [enabled: false], + properties: [row_id: properties.row_id, a: properties.a, n: properties.n]]]) + } request("PUT", "${disabled}/_doc/1?refresh=true", [row_id: 1, a: "ok", n: 1]) request("POST", "_aliases", [actions: [[add: [index: index, alias: "doris_source_disabled_alias_${kind}"]], [add: [index: disabled, alias: "doris_source_disabled_alias_${kind}"]]]]) @@ -130,7 +146,8 @@ suite("test_es_source_fallback_types", "p2,external") { sql "insert into es_retained_not_expected values (1,null,'miss'),(2,1,'miss'),(3,2,'hit'),(4,2,'miss')" def notExpected = "internal.${context.dbName}.es_retained_not_expected" def originalNotRules = sql("show variables like 'disable_nereids_expression_rules'")[0][1] - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] def originalZone = sql("show variables like 'time_zone'")[0][1] try { diff --git a/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy b/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy index 831636f700374d..4466a4ac77b00c 100644 --- a/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_source_semantics.groovy @@ -184,7 +184,8 @@ suite("test_es_source_semantics", "p2,external") { "sibling like '[%'", "sibling not in ('cat')", "not (sibling = 'cat')", "sibling = 'cat' or row_id = 1", "sibling = 'cat' and row_id > 0", "not (sibling = 'cat' and row_id = 1)"] - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] def originalBatch = sql("show variables like 'batch_size'")[0][1] try { diff --git a/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy b/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy index e1b846b9faa61a..8810d6114bd00c 100644 --- a/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_source_value_boundaries.groovy @@ -104,7 +104,8 @@ suite("test_es_source_value_boundaries", "p2,external") { distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" sql "insert into es_inherited_mapping_expected values (1,'VALUE',5,5),(2,'VALUE',5,5)" def mappingExpected = "internal.${context.dbName}.es_inherited_mapping_expected" - def originalV2 = sql("show variables like 'enable_file_scanner_v2'")[0][1] + // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. + def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] def originalZone = sql("show variables like 'time_zone'")[0][1] try { From 08650cbcb7420772b62e8a6a8a4128040bb23c07 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 14:23:45 +0800 Subject: [PATCH 4/7] [fix](be) Fix ORC condition-cache seeks past filtered stripe suffixes (#70079) (cherry picked from commit 75ab5ba108293af01da96e84f612e0b05bdce9db) --- be/test/format_v2/orc/orc_reader_test.cpp | 66 +++++++++++ contrib/apache-orc | 2 +- .../GenerateOrcConditionCacheSeek.cpp | 51 ++++++++ .../tvf/test_data/condition_cache_seek.orc | Bin 0 -> 1458 bytes .../tvf/test_orc_condition_cache_seek.groovy | 110 ++++++++++++++++++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/external_table_p0/tvf/test_data/GenerateOrcConditionCacheSeek.cpp create mode 100644 regression-test/data/external_table_p0/tvf/test_data/condition_cache_seek.orc create mode 100644 regression-test/suites/external_table_p0/tvf/test_orc_condition_cache_seek.groovy diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 2e9e879cf2c682..515081ce73c445 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -6372,6 +6372,72 @@ TEST_F(NewOrcReaderTest, ConditionCacheHitSkipsFalseGranulesBeforeColumnRead) { EXPECT_EQ(rows, 0); } +TEST_F(NewOrcReaderTest, ConditionCacheHitSeeksPastSargFilteredStripeSuffix) { + const auto file_path = find_repo_file( + "regression-test/data/external_table_p0/tvf/test_data/condition_cache_seek.orc"); + ASSERT_FALSE(file_path.empty()); + for (const bool lazy : {false, true}) { + for (const bool min_max : {false, true}) { + SCOPED_TRACE(fmt::format("lazy={}, min_max={}", lazy, min_max)); + auto io_ctx = std::make_shared(); + auto reader = create_reader_for_path(file_path.string(), nullptr, io_ctx); + TQueryOptions options; + options.__set_enable_orc_lazy_mat(lazy); + options.__set_enable_orc_filter_by_min_max(min_max); + RuntimeState state {options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + reader->set_batch_size(ConditionCacheContext::GRANULE_SIZE); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(1), format::LocalIndex(1)}}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, 5000))); + ASSERT_TRUE(reader->open(request).ok()); + + auto ctx = std::make_shared(); + ctx->is_hit = true; + ctx->filter_result = std::make_shared>(20, false); + (*ctx->filter_result)[0] = true; + (*ctx->filter_result)[9] = true; + reader->set_condition_cache_context(ctx); + int seeks = 0; + ScopedDebugPoint debug_point( + "OrcReader._skip_condition_cache_false_granules.before_seek_to_row", + [&]() { ++seeks; }); + + // Granule 9 contains the next stripe's matching row, but starts in the + // current stripe's filtered suffix. Both columns must resume in stripe 2. + std::vector ids; + std::vector payloads; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto& id_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + const auto& payload_column = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + payloads.push_back(payload_column.get_element(row)); + } + } + EXPECT_EQ(ids, (std::vector {10000, 10001})); + EXPECT_EQ(payloads, (std::vector {0, 20000})); + EXPECT_GT(seeks, 0); + EXPECT_GT(io_ctx->condition_cache_filtered_rows, 0); + } + } +} + TEST_F(NewOrcReaderTest, ConditionCacheSeekReturnsCleanEofWhenCancelled) { constexpr int64_t row_count = ConditionCacheContext::GRANULE_SIZE * 2; const auto file_path = (_test_dir / "condition_cache_cancelled_seek.orc").string(); diff --git a/contrib/apache-orc b/contrib/apache-orc index be0f1b73a7aeb7..5962bb071d6345 160000 --- a/contrib/apache-orc +++ b/contrib/apache-orc @@ -1 +1 @@ -Subproject commit be0f1b73a7aeb78824a03e0dcb692c50a176d513 +Subproject commit 5962bb071d6345cc29c5353d64d85a203525e4d9 diff --git a/regression-test/data/external_table_p0/tvf/test_data/GenerateOrcConditionCacheSeek.cpp b/regression-test/data/external_table_p0/tvf/test_data/GenerateOrcConditionCacheSeek.cpp new file mode 100644 index 00000000000000..be4ec1bbb61486 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_data/GenerateOrcConditionCacheSeek.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +// Link against the ORC library used by Doris and pass the output fixture path. +// Each stripe has 20,000 rows and two 10,000-row index groups. Only its first row +// matches id > 5000. The second match makes cache granule [18432, 20480) true, +// so a cache hit seeks into the first stripe's SARG-filtered suffix at row 18432. +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + auto stream = orc::writeLocalFile(argv[1]); + auto type = std::unique_ptr( + orc::Type::buildTypeFromString("struct")); + orc::WriterOptions options; + options.setStripeSize(1).setRowIndexStride(10000).setCompression(orc::CompressionKind_NONE); + auto writer = orc::createWriter(*type, stream.get(), options); + auto batch = writer->createRowBatch(20000); + auto& root = dynamic_cast(*batch); + auto& ids = dynamic_cast(*root.fields[0]); + auto& payloads = dynamic_cast(*root.fields[1]); + for (int64_t stripe = 0; stripe < 2; ++stripe) { + for (int64_t row = 0; row < 20000; ++row) { + ids.data[row] = row == 0 ? 10000 + stripe : 0; + payloads.data[row] = stripe * 20000 + row; + } + root.numElements = ids.numElements = payloads.numElements = 20000; + writer->add(*batch); + } + writer->close(); + stream->close(); + return 0; +} diff --git a/regression-test/data/external_table_p0/tvf/test_data/condition_cache_seek.orc b/regression-test/data/external_table_p0/tvf/test_data/condition_cache_seek.orc new file mode 100644 index 0000000000000000000000000000000000000000..3cb2b0524c0b290180653ca07fc6dcbd0e369a88 GIT binary patch literal 1458 zcmd7SKWGzC90%}s@2<)7Ov8CpeMZrcDT{3shgz4G4rVbq+73=)q=SlJakAiWC_*QD zgF^->K^!t>aL6F#>Xb#uP$Yxsfr9}-4pa;ljKAMEwSi#v4)=L4zxU@}xbIzgY0*x) z6DlZqV%RxpOCj82RCR47oZ1UR(?v6;Y}4D6?j(;SXhdj1RgLK`7mreLi>All-_Ov< z8QZJ`(ml>YX+?WD+fAwCUr#q_w;Q6A-RTQ)Sg-Ib&L(&IzG56kGg&6j6qs3No+&aO zbDg=#+-BC8hm3Eq8!&)XSc7%gfK3>}7Hq=~j9?eWum^pK{b2yBumQ9Di@#7}tKQrdlYb(Rd9HeL@JTEB?bd3$qX@qA# zAz@QGA9qb1@4f!UODI1NhWUwhP3KQ+>A3&-okV_ehc!EVu0ek2FVO=DL)e0C*ntu3 z!Wj0TZ}O@Q7{Ds5!8&ZfCJbQ#LO<_>24XCWjT-Z^ZnnsQ!3?3LuTZi)l71&vVQM=Wi>1Eq9Afg znFl><&@0d_<`R4(Pt3X8;`n@&x2R2_#>eM_gkOICp76wcP%0X?(if&LPZ!QFmlnL` cVtGYv8T4ZMzLk?AmA_j^JpN&t<)uaa7bH^vqW}N^ literal 0 HcmV?d00001 diff --git a/regression-test/suites/external_table_p0/tvf/test_orc_condition_cache_seek.groovy b/regression-test/suites/external_table_p0/tvf/test_orc_condition_cache_seek.groovy new file mode 100644 index 00000000000000..e6a056f5747d46 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_orc_condition_cache_seek.groovy @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.azure.storage.blob.BlobClientBuilder +import com.azure.storage.common.StorageSharedKeyCredential +import org.apache.doris.regression.action.ProfileAction + +suite("test_orc_condition_cache_seek", "p0,external") { + String provider = getS3Provider() + String endpoint = getS3Endpoint() + if (provider.equalsIgnoreCase("AZURE")) { + if (endpoint == "azure-endpoint") { + endpoint = "https://${getS3AK()}.blob.core.windows.net" + } else if (!endpoint.contains("://")) { + endpoint = "https://${endpoint}" + } + } + String bucket = context.config.otherConfigs.get("s3BucketName") + String key = "orc_condition_cache_seek/${UUID.randomUUID()}/values.orc" + File fixture = new File(context.dataPath, "test_data/condition_cache_seek.orc") + def blobClient = null + def s3Client = null + boolean uploaded = false + Map originalSettings = [:] + for (String variable : ["enable_condition_cache", "enable_orc_lazy_materialization", + "enable_orc_filter_by_min_max", "batch_size", "enable_sql_cache", + "enable_query_cache", "enable_profile", "profile_level", "parallel_pipeline_task_num", + "max_file_scanners_concurrency"]) { + originalSettings[variable] = sql("SHOW VARIABLES LIKE '${variable}'")[0][1].toString() + } + try { + if (provider.equalsIgnoreCase("AZURE")) { + blobClient = new BlobClientBuilder().endpoint(endpoint).containerName(bucket) + .blobName(key).credential(new StorageSharedKeyCredential(getS3AK(), getS3SK())) + .buildClient() + blobClient.uploadFromFile(fixture.absolutePath) + } else { + s3Client = getS3Client() + s3Client.putObject(bucket, key, fixture) + } + uploaded = true + String source = """S3('uri'='s3://${bucket}/${key}', 'format'='orc', + 'provider'='${provider}', 's3.endpoint'='${endpoint}', 's3.region'='${getS3Region()}', + 's3.access_key'='${getS3AK()}', 's3.secret_key'='${getS3SK()}')""" + sql "SET enable_sql_cache = false" + sql "SET enable_query_cache = false" + sql "SET enable_profile = true" + sql "SET profile_level = 2" + sql "SET parallel_pipeline_task_num = 1" + sql "SET max_file_scanners_concurrency = 1" + sql "SET batch_size = 2048" + // A full LRU-K shard needs one scan for ghost history and another for admission. + // Allow a third visit to one backend even when scans are spread across all backends. + int attempts = 2 * sql("SHOW BACKENDS").size() + 1 + for (boolean minMax : [false, true]) { + sql "SET enable_orc_filter_by_min_max = ${minMax}" + for (boolean lazy : [false, true]) { + sql "SET enable_orc_lazy_materialization = ${lazy}" + sql "SET enable_condition_cache = false" + quickTest("uncached", "SELECT id, payload FROM ${source} WHERE id > 5000 ORDER BY id", + "SELECT 10000, CAST(0 AS BIGINT) UNION ALL SELECT 10001, CAST(20000 AS BIGINT) ORDER BY 1") + sql "SET enable_condition_cache = true" + boolean cacheHit = false + for (int attempt = 0; attempt < attempts && !cacheHit; ++attempt) { + String token = "orc_seek_${UUID.randomUUID().toString().replace('-', '')}" + // The first match populates granule 0. The next stripe's match populates + // granule 9, whose start (18432) lies in the first stripe's pruned suffix. + quickTest("cached", + "SELECT id, payload FROM (SELECT id, payload, '${token}' AS marker " + + "FROM ${source} WHERE id > 5000) t ORDER BY id", + "SELECT 10000, CAST(0 AS BIGINT) UNION ALL " + + "SELECT 10001, CAST(20000 AS BIGINT) ORDER BY 1") + String profile = new ProfileAction(context).getProfileBySql(token, + ["ConditionCacheHit", "ConditionCacheFilteredRows"]) + cacheHit = (profile =~ /(?m)ConditionCacheHit:\s*[1-9]/).find() + if (cacheHit) { + assertTrue((profile =~ /(?m)ConditionCacheFilteredRows:\s*[1-9]/).find(), profile) + } + } + assertTrue(cacheHit, "No condition cache hit after ${attempts} scans") + } + } + } finally { + try { + originalSettings.each { variable, value -> sql "SET ${variable} = ${value}" } + } finally { + if (uploaded) { + if (blobClient != null) { + blobClient.delete() + } else { + s3Client.deleteObject(bucket, key) + } + } + } + } +} From 2afb57fb3966f9a4ee91c2185862f67bce7ea92e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 14:25:40 +0800 Subject: [PATCH 5/7] [fix](be) Preserve JSON and WAL rows with empty file projections (#70154) (cherry picked from commit 4bde8d0fb28452c9bb8314febbe05c4d63dde93c) --- be/src/format_v2/json/json_reader.cpp | 53 ++++-- be/src/format_v2/json/json_reader.h | 3 + be/src/format_v2/wal/wal_reader.cpp | 5 +- be/test/format_v2/json/json_reader_test.cpp | 162 ++++++++++++++++++ be/test/format_v2/wal/wal_reader_test.cpp | 120 +++++++++++++ .../test_group_commit_wal_row_count.out | 17 ++ .../hive/test_hive_json_partition_only.groovy | 85 +++++++++ .../test_group_commit_wal_row_count.groovy | 85 +++++++++ 8 files changed, 517 insertions(+), 13 deletions(-) create mode 100644 regression-test/data/insert_p0/group_commit/test_group_commit_wal_row_count.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_json_partition_only.groovy create mode 100644 regression-test/suites/insert_p0/group_commit/test_group_commit_wal_row_count.groovy diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 3d351864087710..35d626b0175d0d 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -384,14 +384,14 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { return Status::InternalError("JSON v2 reader is not open"); } - const auto batch_size = _runtime_state->batch_size(); + const auto batch_size = cast_set(_runtime_state->batch_size()); const auto max_block_bytes = _runtime_state->preferred_block_size_bytes(); *rows = 0; *eof = false; + _logical_rows = 0; _missing_fields.clear(); - while (file_block->rows() < batch_size && !_reader_eof && - file_block->bytes() < max_block_bytes) { + while (_logical_rows < batch_size && !_reader_eof && file_block->bytes() < max_block_bytes) { if (_read_json_by_line && _skip_first_line) { RETURN_IF_ERROR(_line_reader->skip_split_prefix( _reader_range.start_offset, _line_delimiter, &_reader_eof, _io_ctx.get())); @@ -399,7 +399,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { continue; } - const size_t original_rows = file_block->rows(); + const size_t original_rows = _logical_rows; size_t size = 0; bool is_empty_row = false; Status st = Status::OK(); @@ -428,12 +428,13 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } // An ignored or empty JSON object can produce no row. Avoid spinning forever on a document // that was consumed but produced no materialized value. - if (!is_empty_row && file_block->rows() == original_rows) { + if (!is_empty_row && _logical_rows == original_rows) { break; } } - *rows = file_block->rows(); + *rows = _logical_rows; + DCHECK(file_block->columns() == 0 || file_block->rows() == *rows); for (auto& [_, mask] : _missing_fields) { mask.resize(*rows, 0); } @@ -794,7 +795,7 @@ Status JsonReader::_append_simple_json_rows(Block* block, bool* is_empty_row, bo _array_iter = _array.begin(); while (_array_iter != _array.end()) { simdjson::ondemand::object object_value = (*_array_iter).get_object(); - const size_t row_start = block->rows(); + const size_t row_start = _logical_rows; const auto status = _set_column_values_from_object(&object_value, block, &valid); if (!status.ok()) { if (!_request->reject_invalid_rows) { @@ -825,12 +826,12 @@ Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty DORIS_CHECK(block != nullptr); DORIS_CHECK(is_empty_row != nullptr); DORIS_CHECK(eof != nullptr); - const size_t original_rows = block->rows(); + const size_t original_rows = _logical_rows; bool valid = true; _array = _json_value.get_array(); _array_iter = _array.begin(); while (_array_iter != _array.end()) { - const size_t row_start = block->rows(); + const size_t row_start = _logical_rows; simdjson::ondemand::object object_value = (*_array_iter).get_object(); if (!_parsed_from_json_root && !_parsed_json_root.empty()) { // Reject roots per element for loads, preserving earlier siblings and their presence @@ -869,7 +870,7 @@ Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty } ++_array_iter; } - *is_empty_row = block->rows() == original_rows; + *is_empty_row = _logical_rows == original_rows; return Status::OK(); } @@ -893,7 +894,7 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob DORIS_CHECK(block != nullptr); DORIS_CHECK(valid != nullptr); std::vector seen_columns(block->columns(), false); - const size_t cur_row_count = block->rows(); + const size_t cur_row_count = _logical_rows; bool has_valid_value = false; size_t key_index = 0; @@ -944,21 +945,46 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob } } *valid = true; + ++_logical_rows; if (!has_valid_value) { return Status::OK(); } return Status::OK(); } +Status JsonReader::_has_matching_jsonpath(simdjson::ondemand::object* object_value, + bool* matches) const { + *matches = false; + for (const auto& path : _parsed_jsonpaths) { + // A root path selects the current document without a field lookup. + if (JsonFunctions::is_root_path(path)) { + *matches = true; + continue; + } + simdjson::ondemand::value value; + const auto status = JsonFunctions::extract_from_object(*object_value, path, &value); + if (!status.ok() && !status.is()) { + return status; + } + *matches = *matches || status.ok(); + } + return Status::OK(); +} + Status JsonReader::_write_columns_by_jsonpath(simdjson::ondemand::object* object_value, Block* block, bool* valid) { DORIS_CHECK(object_value != nullptr); DORIS_CHECK(block != nullptr); DORIS_CHECK(valid != nullptr); bool has_valid_value = false; - const size_t cur_row_count = block->rows(); + const size_t cur_row_count = _logical_rows; _cached_string_values.clear(); + // Even without projected fields, jsonpaths still decides whether the object contributes a row. + if (_requested_columns.empty()) { + RETURN_IF_ERROR(_has_matching_jsonpath(object_value, &has_valid_value)); + } + for (size_t i = 0; i < _requested_columns.size(); ++i) { const auto& requested = _requested_columns[i]; auto* column_ptr = block->get_by_position(i).column->assert_mutable().get(); @@ -1015,6 +1041,7 @@ Status JsonReader::_write_columns_by_jsonpath(simdjson::ondemand::object* object return Status::OK(); } *valid = true; + ++_logical_rows; return Status::OK(); } @@ -1217,6 +1244,7 @@ Status JsonReader::_append_null_for_malformed_json(Block* block) { assert_cast(column.get())->insert_default(); column_with_type.column = std::move(column); } + ++_logical_rows; return Status::OK(); } @@ -1228,6 +1256,7 @@ Status JsonReader::_handle_json_error(const Status& status, Block* block, size_t // block to the row count before this document before either surfacing the error or appending // the ignore-malformed null row. _truncate_block_to_rows(block, original_rows); + _logical_rows = original_rows; for (auto& [_, mask] : _missing_fields) { mask.resize(original_rows, 0); } diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index ba2a0d99e2f2a6..b1c413da0e7551 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -148,6 +148,7 @@ class JsonReader final : public FileReader { Status _append_nested_jsonpath_row(Block* block, bool* is_empty_row, bool* eof); Status _set_column_values_from_object(simdjson::ondemand::object* object_value, Block* block, bool* valid); + Status _has_matching_jsonpath(simdjson::ondemand::object* object_value, bool* matches) const; Status _write_columns_by_jsonpath(simdjson::ondemand::object* object_value, Block* block, bool* valid); template @@ -191,6 +192,8 @@ class JsonReader final : public FileReader { std::unique_ptr _decompressor; std::unique_ptr _line_reader; int64_t _current_offset = 0; + // Partition-only scans have no file columns, so Block::rows() cannot count their records. + size_t _logical_rows = 0; bool _reader_eof = false; bool _skip_first_line = false; bool _single_document_read = false; diff --git a/be/src/format_v2/wal/wal_reader.cpp b/be/src/format_v2/wal/wal_reader.cpp index 55e88796c9805e..5f57a422ff8fc2 100644 --- a/be/src/format_v2/wal/wal_reader.cpp +++ b/be/src/format_v2/wal/wal_reader.cpp @@ -195,8 +195,11 @@ Status WalReader::get_block(Block* file_block, size_t* rows, bool* eof) { return Status::Corruption("WAL block has {} columns but header declares {}", source_block.columns(), _column_ids.size()); } + // Empty projections still contribute every WAL row. Capture the count before materialization + // moves source columns into the output block and can leave the source with no owned columns. + const size_t source_rows = source_block.rows(); RETURN_IF_ERROR(_materialize_requested_columns(&source_block, file_block)); - *rows = file_block->rows(); + *rows = source_rows; _record_scan_rows(cast_set(*rows)); RETURN_IF_ERROR( apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index b14f68b1c2c5ca..ff4adc226ac1d9 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -43,6 +43,7 @@ #include "exprs/vexpr_context.h" #include "format/json/new_json_reader.h" #include "format_v2/column_data.h" +#include "format_v2/table_reader.h" #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/runtime_profile.h" @@ -856,4 +857,165 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); } +// Partition-only scans have no file-local output columns, but still need one logical row per +// accepted JSON record so TableReader can materialize the partition constants. +TEST(JsonReaderTest, EmptyFileProjectionPreservesRows) { + ObjectPool pool; + const auto slots = build_slots(&pool); + for (const auto& source_slots : {slots, std::vector {}}) { + auto result = read_once("empty_projection.jsonl", "{\"id\":1}\n\n{}\n{\"id\":3}\n", + json_scan_params(), source_slots, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.block.columns(), 0); + EXPECT_EQ(result.rows, 3); + EXPECT_FALSE(result.eof); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_EQ(result.second_rows, 0); + EXPECT_TRUE(result.second_eof); + } +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesOuterArrayRows) { + auto result = read_once("empty_projection_array.json", R"([{"id":1},{},{"id":3}])", + json_scan_params(false, true), {}, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 3); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesJsonRootRows) { + auto result = read_once("empty_projection_root.jsonl", + "{\"payload\":{\"id\":1}}\n{\"payload\":{\"id\":2}}\n", + json_scan_params(true, false, "", "$.payload"), {}, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 2); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, EmptyFileProjectionHonorsJsonPaths) { + ObjectPool pool; + const auto slots = build_slots(&pool); + auto result = read_once("empty_projection_jsonpaths.jsonl", + "{\"payload\":{\"id\":1}}\n{\"other\":2}\n{\"payload\":{\"id\":3}}\n", + json_scan_params(true, false, R"(["$.payload.id","$.payload.user"])"), + slots, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 2); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesRootJsonPathRows) { + auto result = read_once("empty_projection_root_path.jsonl", "{\"id\":1}\n{}\n", + json_scan_params(true, false, R"(["$."])"), {}, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 2); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesRootedArrayJsonPathRows) { + auto result = read_once("empty_projection_rooted_array.json", + R"({"rows":[{"payload":{"id":1}},{"other":2},{"payload":{"id":3}}]})", + json_scan_params(false, true, R"(["$.payload.id"])", "$.rows"), {}, {}, + true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 2); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_TRUE(result.second_eof); +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesMalformedNullRows) { + auto result = read_once("empty_projection_malformed.jsonl", "not-json\n{\"id\":1}\n", + json_scan_params(true, false, "", "", true), {}, {}, true, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 2); + EXPECT_TRUE(result.second_status.ok()); + EXPECT_TRUE(result.second_eof); + + auto strict = read_once("empty_projection_strict.jsonl", "not-json\n", json_scan_params(), {}, + {}, false, true); + EXPECT_FALSE(strict.status.ok()); +} + +TEST(JsonReaderTest, EmptyFileProjectionPreservesEmptyInput) { + for (const std::string content : {"", "\n", "[]\n"}) { + auto result = read_once("empty_projection_empty.jsonl", content, + json_scan_params(true, true), {}, {}, false, true); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_EQ(result.rows, 0); + EXPECT_TRUE(result.eof); + } +} + +TEST(JsonReaderTest, EmptyFileProjectionHonorsBatchSize) { + const auto path = write_json_file("empty_projection_batches.jsonl", + "{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n" + "{\"id\":4}\n{\"id\":5}\n"); + auto properties = std::make_shared(); + properties->system_type = TFileType::FILE_LOCAL; + auto desc = file_description(path.string()); + auto params = json_scan_params(); + RuntimeProfile profile("json_empty_projection_batches"); + MockRuntimeState state; + state._batch_size = 2; + JsonReader reader(properties, desc, std::make_shared(), &profile, ¶ms, + file_range(path), {}); + ASSERT_TRUE(reader.init(&state).ok()); + ASSERT_TRUE(reader.open(std::make_shared()).ok()); + for (const size_t expected_rows : {2, 2, 1, 0}) { + Block block; + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(block.columns(), 0); + EXPECT_EQ(rows, expected_rows); + EXPECT_EQ(eof, expected_rows == 0); + } +} + +TEST(JsonReaderTest, EmptyFileProjectionMaterializesPartitionRows) { + const auto path = + write_json_file("partition_only.jsonl", "{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"); + auto params = json_scan_params(); + const std::vector file_slots; + ColumnDefinition partition; + partition.identifier = Field::create_field("dt"); + partition.local_id = 0; + partition.name = "dt"; + partition.type = std::make_shared(); + partition.is_partition_key = true; + RuntimeProfile profile("json_partition_only"); + MockRuntimeState state; + state._batch_size = 2; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = {partition}, + .conjuncts = {}, + .format = FileFormat::JSON, + .scan_params = ¶ms, + .io_ctx = std::make_shared(), + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &file_slots}) + .ok()); + SplitReadOptions split; + split.current_range = file_range(path); + split.current_split_format = FileFormat::JSON; + split.partition_values.emplace("dt", Field::create_field("p1")); + ASSERT_TRUE(reader.prepare_split(split).ok()); + bool eos = false; + for (const size_t expected_rows : {2, 1, 0}) { + Block block({{partition.type->create_column(), partition.type, partition.name}}); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_EQ(block.rows(), expected_rows); + EXPECT_EQ(eos, expected_rows == 0); + const auto column = block.get_by_position(0).column->convert_to_full_column_if_const(); + for (size_t row = 0; row < expected_rows; ++row) { + EXPECT_EQ(column->get_data_at(row).to_string(), "p1"); + } + } + EXPECT_TRUE(reader.close().ok()); +} + } // namespace doris::format::json diff --git a/be/test/format_v2/wal/wal_reader_test.cpp b/be/test/format_v2/wal/wal_reader_test.cpp index 77c14afb12f1ab..65e37bd5f89774 100644 --- a/be/test/format_v2/wal/wal_reader_test.cpp +++ b/be/test/format_v2/wal/wal_reader_test.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include "agent/be_exec_version_manager.h" #include "core/block/block.h" @@ -60,8 +62,126 @@ PBlock serialize_block(const Block& block) { return pblock; } +void expect_wal_column(const IColumn& column, const std::string& name, size_t expected_rows, + size_t offset) { + ASSERT_EQ(column.size(), expected_rows); + for (size_t row = 0; row < expected_rows; ++row) { + EXPECT_EQ(column.get_data_at(row).to_string(), name + std::to_string(offset + row)); + } +} + +void expect_wal_batch(WalReader* reader, Block* output, size_t expected_columns, + size_t expected_rows, size_t offset) { + output->clear_column_data(); + size_t rows = 0; + bool eof = true; + ASSERT_TRUE(reader->get_block(output, &rows, &eof).ok()); + // Projection can remove every output column while the WAL block still contributes rows. + EXPECT_EQ(rows, expected_rows); + EXPECT_FALSE(eof); + ASSERT_EQ(output->columns(), expected_columns); + for (const auto& column : output->get_columns_with_type_and_name()) { + ASSERT_NO_FATAL_FAILURE( + expect_wal_column(*column.column, column.name, expected_rows, offset)); + } +} + +void initialize_wal_reader(WalReader* reader, const std::string& path, bool discover_schema) { + reader->_wal_reader = std::make_shared(path); + ASSERT_TRUE(reader->_wal_reader->init().ok()); + std::string encoded_ids; + ASSERT_TRUE(reader->_wal_reader->read_header(reader->_version, encoded_ids).ok()); + ASSERT_TRUE(parse_wal_column_ids(encoded_ids, &reader->_column_ids).ok()); + if (discover_schema) { + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + } +} + +void expect_wal_eof(WalReader* reader, Block* output) { + output->clear_column_data(); + size_t rows = 99; + bool eof = false; + ASSERT_TRUE(reader->get_block(output, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + EXPECT_EQ(output->rows(), 0); +} + +class WalReaderV2ProjectionTest + : public testing::TestWithParam, bool>> { +protected: + void SetUp() override { + _wal_path = temporary_wal_path(); + const auto type = std::make_shared(); + WalWriter writer(_wal_path); + ASSERT_TRUE(writer.init(io::global_local_filesystem()).ok()); + ASSERT_TRUE(writer.append_header("17,4").ok()); + size_t offset = 0; + for (const size_t batch_rows : {3, 2}) { + Block source; + for (const auto& name : {"left_value", "right_value"}) { + auto column = type->create_column(); + for (size_t row = 0; row < batch_rows; ++row) { + const auto value = std::string(name) + std::to_string(offset + row); + column->insert_data(value.data(), value.size()); + } + source.insert({std::move(column), type, name}); + } + auto pblock = serialize_block(source); + ASSERT_TRUE(writer.append_blocks({&pblock}).ok()); + offset += batch_rows; + } + ASSERT_TRUE(writer.finalize().ok()); + } + + void TearDown() override { + std::filesystem::remove_all( + std::filesystem::path(_wal_path).parent_path().parent_path().parent_path()); + } + + std::string _wal_path; +}; + } // namespace +TEST_P(WalReaderV2ProjectionTest, PreservesRowsAcrossBlocksAndEof) { + const auto& [projected_columns, discover_schema] = GetParam(); + std::shared_ptr properties; + std::unique_ptr description; + WalReader reader(properties, description, nullptr, nullptr, {}); + ASSERT_NO_FATAL_FAILURE(initialize_wal_reader(&reader, _wal_path, discover_schema)); + + auto request = std::make_shared(); + Block output; + const auto type = std::make_shared(); + const std::vector names {"left_value", "right_value"}; + for (size_t position = 0; position < projected_columns.size(); ++position) { + const auto source_position = projected_columns[position]; + request->local_positions.emplace(LocalColumnId(source_position), LocalIndex(position)); + output.insert({type->create_column(), type, names[source_position]}); + } + ASSERT_TRUE(reader.open(std::move(request)).ok()); + + size_t offset = 0; + for (const size_t expected_rows : {3, 2}) { + ASSERT_NO_FATAL_FAILURE(expect_wal_batch(&reader, &output, projected_columns.size(), + expected_rows, offset)); + offset += expected_rows; + } + for (int attempt = 0; attempt < 2; ++attempt) { + ASSERT_NO_FATAL_FAILURE(expect_wal_eof(&reader, &output)); + } + ASSERT_TRUE(reader.close().ok()); +} + +INSTANTIATE_TEST_SUITE_P(WriterBacked, WalReaderV2ProjectionTest, + testing::Combine(testing::Values(std::vector {}, std::vector {0}, + std::vector {1}, + std::vector {1, 0}), + testing::Bool())); + TEST(WalReaderV2Test, ParseColumnIdsPreservesHeaderOrder) { std::vector column_ids; ASSERT_TRUE(parse_wal_column_ids("17,4,99", &column_ids).ok()); diff --git a/regression-test/data/insert_p0/group_commit/test_group_commit_wal_row_count.out b/regression-test/data/insert_p0/group_commit/test_group_commit_wal_row_count.out new file mode 100644 index 00000000000000..0ae4de993337b0 --- /dev/null +++ b/regression-test/data/insert_p0/group_commit/test_group_commit_wal_row_count.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_replay -- +0 + +-- !counts -- +5 4 15 120 + +-- !values -- +1 10 +2 20 +3 \N +4 40 +5 50 + +-- !filtered_count -- +3 + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_json_partition_only.groovy b/regression-test/suites/external_table_p0/hive/test_hive_json_partition_only.groovy new file mode 100644 index 00000000000000..69ae572d563a10 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_json_partition_only.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_json_partition_only", "p0,external,hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String catalog = "test_hive_json_partition_only" + def previousSettings = sql """SELECT @@enable_count_push_down_for_external_table, + @@enable_sql_cache""" + sql """DROP CATALOG IF EXISTS ${catalog}""" + try { + sql """CREATE CATALOG ${catalog} PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://${externalEnvIp}:${hmsPort}" + )""" + sql """USE `${catalog}`.`default`""" + sql "SET enable_sql_cache=false" + + // Reading a real JSON field provides the row-count oracle for the same partition files. + def baseline = sql "SELECT id, dt FROM json_all_complex_types" + assertTrue(!baseline.isEmpty()) + def partitions = baseline.groupBy { row -> row[1] } + + for (boolean pushdown : [false, true]) { + sql "SET enable_count_push_down_for_external_table=${pushdown}" + partitions.each { partition, rows -> + assertNotNull(partition) + String value = partition.toString().replace("'", "''") + String predicate = "dt = '${value}'" + String countQuery = "SELECT COUNT(*) FROM json_all_complex_types WHERE ${predicate}" + explain { + sql countQuery + contains "pushdown agg=NONE" + } + def count = sql countQuery + assertEquals(rows.size(), (count[0][0] as Number).intValue()) + + // Partition values come from the split, so neither query needs a JSON file column. + def projected = sql "SELECT dt FROM json_all_complex_types WHERE ${predicate}" + assertEquals(rows.size(), projected.size()) + assertTrue(projected.every { row -> row[0] == partition }) + def partitionCount = sql "SELECT COUNT(dt) FROM json_all_complex_types WHERE ${predicate}" + assertEquals(rows.size(), (partitionCount[0][0] as Number).intValue()) + + def filtered = sql """SELECT COUNT(*) FROM json_all_complex_types + WHERE ${predicate} AND id IS NOT NULL""" + assertEquals(rows.count { row -> row[0] != null }, + (filtered[0][0] as Number).intValue()) + } + + def grouped = sql "SELECT dt, COUNT(*) FROM json_all_complex_types GROUP BY dt" + assertEquals(partitions.size(), grouped.size()) + grouped.each { row -> + assertEquals(partitions[row[0]].size(), (row[1] as Number).intValue()) + } + def empty = sql """SELECT COUNT(*) FROM json_all_complex_types + WHERE dt = '__missing_json_partition__'""" + assertEquals(0, (empty[0][0] as Number).intValue()) + } + } finally { + sql "SET enable_count_push_down_for_external_table=${previousSettings[0][0]}" + sql "SET enable_sql_cache=${previousSettings[0][1]}" + sql """DROP CATALOG IF EXISTS ${catalog}""" + } +} diff --git a/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_row_count.groovy b/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_row_count.groovy new file mode 100644 index 00000000000000..ed1a449724a693 --- /dev/null +++ b/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_row_count.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_group_commit_wal_row_count", "nonConcurrent") { + sql "DROP TABLE IF EXISTS test_group_commit_wal_row_count" + sql """ + CREATE TABLE test_group_commit_wal_row_count ( + k INT NOT NULL, + v INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "group_commit_interval_ms" = "2000" + ) + """ + + String loadError = "LoadBlockQueue._finish_group_commit_load.load_error" + String replayStop = "WalTable.replay_wals.stop" + try { + GetDebugPoint().enableDebugPointForAllBEs(loadError) + GetDebugPoint().enableDebugPointForAllBEs(replayStop) + + // Abort each original transaction before the next load so both WALs must be replayed. + for (String data : ["1,10\n2,20\n3,\\N\n", "4,40\n5,50\n"]) { + long txnId = 0 + streamLoad { + table "test_group_commit_wal_row_count" + set "column_separator", "," + set "group_commit", "async_mode" + unset "label" + inputStream new ByteArrayInputStream(data.getBytes("UTF-8")) + time 10000 + check { result, exception, startTime, endTime -> + assertNull(exception) + def response = parseJson(result) + assertEquals("Success", response.Status) + assertEquals(data.readLines().size(), response.NumberTotalRows as int) + txnId = response.TxnId as long + assertTrue(txnId > 0) + } + } + Awaitility.await().atMost(30, SECONDS).pollInterval(1, SECONDS).until { + def txn = sql_return_maparray "SHOW TRANSACTION WHERE ID = ${txnId}" + return txn.size() == 1 && txn[0].TransactionStatus == "ABORTED" + } + } + qt_before_replay "SELECT COUNT(*) FROM test_group_commit_wal_row_count" + + // Keep the original-load failure enabled: rows can become visible only through WAL replay. + GetDebugPoint().disableDebugPointForAllBEs(replayStop) + Awaitility.await().atMost(90, SECONDS).pollInterval(1, SECONDS).until { + def rows = sql "SELECT COUNT(*) FROM test_group_commit_wal_row_count" + return rows[0][0] == 5 + } + qt_counts """SELECT COUNT(*), COUNT(v), SUM(k), SUM(v) + FROM test_group_commit_wal_row_count""" + qt_values "SELECT k, v FROM test_group_commit_wal_row_count ORDER BY k" + qt_filtered_count "SELECT COUNT(*) FROM test_group_commit_wal_row_count WHERE v >= 20" + } finally { + try { + GetDebugPoint().disableDebugPointForAllBEs(replayStop) + } finally { + GetDebugPoint().disableDebugPointForAllBEs(loadError) + } + } +} From d61e482e44f3628bb9f961a0e768d7109609d758 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 17:43:06 +0800 Subject: [PATCH 6/7] [fix](es) Guard Boolean decoding and preserve source scan predicates ### What problem does this PR solve? Related PR: #70185 Problem Summary: Boolean source values such as 1.25, wide integers or objects can reach incompatible RapidJSON getters in scalar and array decoding. Check the exact JSON types before those getters and return the existing scan errors. Explicit date formats can contain epoch_millis, whose session-local date differs from its UTC index date. Classify all date mappings as unsafe for source-mode predicate pushdown while preserving docvalue eligibility. The branch-4.2 catalog adapter also bypasses the legacy EsTable property parser. Forward max_docvalue_fields so catalog-level limits really select source mode, retaining the legacy default and invalid-value handling. ### Release note Fix incorrect filtering and unsafe Boolean value access in Elasticsearch source scans. Honor max_docvalue_fields for Elasticsearch external catalogs. ### Check List (For Author) - Test: FE ES suites (44 tests, 1 environment skip), FE Checkstyle, clang-format 16 and compilation checks for the modified BE source and test. Full BE unit tests and the extended live ES regression suite were not run to completion locally. - Behavior changed: Yes. Invalid Boolean JSON values return scan errors and unsafe source date predicates are evaluated locally. Catalog field limits now take effect. - Does this need documentation: No. --- be/src/exec/es/es_scroll_parser.cpp | 12 ++- be/test/exec/es_scroll_parser_test.cpp | 16 ++++ .../datasource/es/EsExternalCatalog.java | 12 +++ .../doris/datasource/es/EsExternalTable.java | 2 + .../doris/datasource/es/EsMappingUtils.java | 5 +- .../datasource/es/EsExternalTableTest.java | 77 +++++++++++++++++++ .../datasource/es/EsMappingUtilsTest.java | 12 +++ .../test_es_numeric_source_semantics.groovy | 17 +++- 8 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsExternalTableTest.java diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index dd10d5eb53f67f..f70870f7e58bfb 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -713,7 +713,8 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu return Status::OK(); } - if (col.IsNumber()) { + // ignore_malformed can retain 1.25 or wide integers in source; GetInt requires IsInt. + if (col.IsInt()) { val = static_cast::CppType>(col.GetInt()); return Status::OK(); } @@ -729,6 +730,8 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu } const rapidjson::Value& str_col = is_nested_str ? col[0] : col; + // Disabled mappings can preserve objects instead of Boolean strings in source. + RETURN_ERROR_IF_COL_IS_NOT_STRING(str_col, sub_type); const std::string& str_val = str_col.GetString(); size_t val_size = str_col.GetStringLength(); StringParser::ParseResult result; @@ -1080,8 +1083,9 @@ Status ScrollParser::fill_columns(const std::vector& slots, break; } - if (col.IsNumber()) { - int8_t val = static_cast(col.GetInt()); + // ignore_malformed can retain 1.25 or wide integers in source; GetInt requires IsInt. + if (col.IsInt()) { + auto val = static_cast(col.GetInt()); col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); break; } @@ -1101,6 +1105,8 @@ Status ScrollParser::fill_columns(const std::vector& slots, RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true); + // Disabled mappings can preserve objects instead of Boolean strings in source. + RETURN_ERROR_IF_COL_IS_NOT_STRING(str_col, type); const std::string& val = str_col.GetString(); size_t val_size = str_col.GetStringLength(); StringParser::ParseResult result; diff --git a/be/test/exec/es_scroll_parser_test.cpp b/be/test/exec/es_scroll_parser_test.cpp index d0b44c038d1d47..432c7e9923cb8c 100644 --- a/be/test/exec/es_scroll_parser_test.cpp +++ b/be/test/exec/es_scroll_parser_test.cpp @@ -464,6 +464,8 @@ TEST_F(EsScrollParserTest, BooleanSourceRepresentations) { {R"("false")", false}, {R"("true")", true}, {"false", false}, + {"0", false}, + {"1", true}, {"true", true}}) { ScrollParser parser(false); auto status = @@ -510,6 +512,20 @@ TEST_F(EsScrollParserTest, InvalidBooleanSourceIsRejected) { EXPECT_FALSE(parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()).ok()); } +TEST_F(EsScrollParserTest, UnsupportedBooleanSourceTypesAreRejected) { + for (bool array : {false, true}) { + for (const std::string value : + {"1.25", "2147483648", "9223372036854775808", "{}", R"({"x":1})"}) { + SCOPED_TRACE(value); + SCOPED_TRACE(array); + MutableColumnPtr column; + EXPECT_FALSE(read_scalar(TPrimitiveType::BOOLEAN, array ? "[" + value + "]" : value, + &column, array) + .ok()); + } + } +} + // Test timezone pattern matching for ES datetime parsing fix TEST_F(EsScrollParserTest, TestTimezonePatternMatching) { RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java index 60371365c50536..982ea7986c445c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.es; import org.apache.doris.catalog.EsResource; +import org.apache.doris.catalog.EsTable; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; @@ -94,6 +95,17 @@ public boolean enableDocValueScan() { EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE)); } + public int getMaxDocValueFields() { + int defaultLimit = EsTable.getDEFAULT_MAX_DOCVALUE_FIELDS(); + String value = catalogProperty.getOrDefault(EsResource.MAX_DOCVALUE_FIELDS, String.valueOf(defaultLimit)); + // Preserve legacy ES table parsing for stored catalogs with negative or malformed limits. + try { + return Math.max(0, Integer.parseInt(value.trim())); + } catch (NumberFormatException e) { + return defaultLimit; + } + } + public boolean enableKeywordSniff() { return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF, EsResource.KEYWORD_SNIFF_DEFAULT_VALUE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index 6e9e5731f41532..420cf8b6de2fc1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -97,6 +97,8 @@ private EsTable toEsTable() { esTable.setUserName(esCatalog.getUsername()); esTable.setPasswd(esCatalog.getPassword()); esTable.setEnableDocValueScan(esCatalog.enableDocValueScan()); + // This adapter bypasses the legacy EsTable constructor that parses the field limit. + esTable.setMaxDocValueFields(esCatalog.getMaxDocValueFields()); esTable.setEnableKeywordSniff(esCatalog.enableKeywordSniff()); esTable.setNodesDiscovery(esCatalog.enableNodesDiscovery()); esTable.setHttpSslEnabled(esCatalog.enableSsl()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java index f2f54c166fde7a..74454353757782 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java @@ -278,9 +278,10 @@ private static boolean changesSourceValue(JsonNode field, JsonNode mappingDefaul private static boolean requiresSourcePredicateEvaluation(JsonNode field) { String type = field.path("type").asText(); - // A default date may be a date-only source string read as local midnight, unlike its UTC index value. + // Source dates use the session time zone, including epoch_millis in explicit formats, + // so their local value can differ from the UTC value used by an ES predicate. return "scaled_float".equals(type) || "half_float".equals(type) || "alias".equals(type) - || field.hasNonNull("script") || ("date".equals(type) && !field.has("format")); + || field.hasNonNull("script") || "date".equals(type); } private static boolean isSearchable(JsonNode field) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsExternalTableTest.java new file mode 100644 index 00000000000000..8bb785cce9cc87 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsExternalTableTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.Type; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +class EsExternalTableTest { + @Test + void fieldLimitPreservesLegacyDefaults() { + EsExternalCatalog catalog = new EsExternalCatalog(1, "es_catalog", null, + ImmutableMap.of("hosts", "http://127.0.0.1:9200"), ""); + Assertions.assertEquals(EsTable.getDEFAULT_MAX_DOCVALUE_FIELDS(), catalog.getMaxDocValueFields()); + Map limits = ImmutableMap.of("0", 0, "-1", 0, " 3 ", 3, + "invalid", EsTable.getDEFAULT_MAX_DOCVALUE_FIELDS(), + "2147483648", EsTable.getDEFAULT_MAX_DOCVALUE_FIELDS()); + limits.forEach((value, expected) -> { + EsExternalCatalog configured = new EsExternalCatalog(1, "es_catalog", null, + ImmutableMap.of("hosts", "http://127.0.0.1:9200", "max_docvalue_fields", value), ""); + Assertions.assertEquals(expected.intValue(), configured.getMaxDocValueFields(), value); + }); + } + + @Test + void catalogFieldLimitReachesScanAdapter() throws Exception { + EsExternalCatalog catalog = Mockito.spy(new EsExternalCatalog(1, "es_catalog", null, + ImmutableMap.of("hosts", "http://127.0.0.1:9200", "max_docvalue_fields", "1"), "")); + Mockito.doReturn(Mockito.mock(EsRestClient.class)).when(catalog).getEsRestClient(); + EsExternalTable table = new EsExternalTable(2, "es_index", "es_index", catalog, + Mockito.mock(EsExternalDatabase.class)) { + @Override + public List getFullSchema() { + return ImmutableList.of(new Column("a", Type.INT), new Column("b", Type.INT)); + } + + @Override + public Map getColumn2type() { + return ImmutableMap.of("a", "integer", "b", "integer"); + } + }; + try (MockedConstruction ignored = Mockito.mockConstruction(EsMetaStateTracker.class, + (tracker, context) -> Mockito.when(tracker.searchContext()) + .thenReturn(Mockito.mock(SearchContext.class)))) { + Method adapter = EsExternalTable.class.getDeclaredMethod("toEsTable"); + adapter.setAccessible(true); + EsTable adapted = (EsTable) adapter.invoke(table); + Assertions.assertEquals(1, adapted.getMaxDocValueFields()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java index a96e9f7f519827..1c378e5a7033b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java @@ -222,4 +222,16 @@ void indexMalformedDefaultsRespectFieldOverridesAndFieldTypes() { Assertions.assertTrue(context.getDocValueFieldsContext().containsKey("keyword")); } + @Test + void explicitDateFormatsRequireSourcePredicateEvaluation() { + for (String format : Arrays.asList("yyyy-MM-dd||epoch_millis", + "yyyy-MM-dd HH:mm:ss||epoch_millis")) { + String mapping = "{\"test\":{\"mappings\":{\"properties\":{" + + "\"d\":{\"type\":\"date\",\"format\":\"" + format + "\"}}}}}"; + EsFieldContext context = EsMappingUtils.resolveFieldContext( + Collections.singletonList("d"), "test", mapping, null); + Assertions.assertTrue(context.getSourceModeUnsafeFields().contains("d"), format); + Assertions.assertTrue(context.getDocValueFieldsContext().containsKey("d"), format); + } + } } diff --git a/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy index fdb8dabd59835a..0b8395447bf799 100644 --- a/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_numeric_source_semantics.groovy @@ -56,8 +56,10 @@ suite("test_es_numeric_source_semantics", "p2,external") { analysis: [normalizer: [lowercase_value: [type: "custom", filter: ["lowercase"]]]]], mappings: [properties: [row_id: [type: "long"], a: [type: "keyword", normalizer: "lowercase_value"], - b: [type: "scaled_float", scaling_factor: 4], c: [type: "half_float"]]]]) - [[row_id: 1, a: "ok", b: 2.34, c: 2.34], [row_id: 2, a: "empty", b: null, c: null], + b: [type: "scaled_float", scaling_factor: 4], c: [type: "half_float"], + d: [type: "date", format: "yyyy-MM-dd||epoch_millis"]]]]) + [[row_id: 1, a: "ok", b: 2.34, c: 2.34, d: 1704139200000L], + [row_id: 2, a: "empty", b: null, c: null], [row_id: 3, a: "other", b: -2.34, c: -2.34]].each { row -> request("PUT", "${index}/_doc/${row.row_id}", row) } @@ -78,7 +80,9 @@ suite("test_es_numeric_source_semantics", "p2,external") { // SHOW prefixes deprecated variable names; scalar access keeps the compatibility name stable. def originalV2 = sql("SELECT @@enable_file_scanner_v2")[0][0] def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalTimeZone = sql("select @@time_zone")[0][0] try { + sql "set time_zone = '+08:00'" ["enabled", "disabled", "limited"].each { mode -> def catalog = "es_numeric_source_${mode}" sql "drop catalog if exists ${catalog}" @@ -93,6 +97,14 @@ suite("test_es_numeric_source_semantics", "p2,external") { [false, true].each { parallel -> sql "set enable_es_parallel_scroll = ${parallel}" def tag = "${mode}_v2_${scannerV2}_parallel_${parallel}" + // The epoch is January 1 in ES's UTC index, but January 2 when source is + // decoded in this session. Both predicates must compare the returned date. + assertEquals([], sql("select row_id, a, d from ${index} " + + "where d = cast('2024-01-01' as date)")) + assertEquals([["1", "ok", "2024-01-02"]], + sql("select row_id, a, d from ${index} " + + "where d = cast('2024-01-02' as date)") + .collect { row -> row.collect { it.toString() } }) // The normalizer forces the whole projection to source. Numeric predicates must // compare the original values, even when the indexed representation is rounded. predicates.eachWithIndex { predicate, i -> @@ -138,5 +150,6 @@ suite("test_es_numeric_source_semantics", "p2,external") { } finally { sql "set enable_file_scanner_v2 = ${originalV2}" sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalTimeZone}'" } } From b19a66ec0926ea1ddb5191123bf75d24a6d3f123 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 4 Nov 2026 20:06:33 +0800 Subject: [PATCH 7/7] [fix](external) Address ES source semantics and ARM ORC linking ### What problem does this PR solve? Related PR: #70185, #70205 Problem Summary: Keep standard numeric and JSONB predicates local when ES source values differ from indexed values. Select typed alias backings consistently for inference and scans. Carry per-backing date formats through both scanner protocols and honor calendar/epoch alternatives for strings and source numbers. Link libdeflate on ARM for the pinned ORC decoder. ### Release note Correct Elasticsearch source filtering, mapped date decoding and typed alias selection. Restore the ARM ORC decoder link dependency. Source-mode numeric filters may fetch more rows to preserve Doris value semantics. ### Check List (For Author) - Test: 48 FE tests, 0 failures/errors, 1 environment skip; new FE failures reproduced before fixes. 21 standalone shared date-format cases passed ASAN. Three affected BE translation units compiled. FE Checkstyle, clang-format 16 and regression-suite compilation passed. CMake dependency checks passed for aarch64, arm64 and x86_64. Full BE tests, ARM linking and live ES regression remain for CI. - Behavior changed: Yes, source predicates and date parsing now follow the materialized value and mapped grammar; docvalue eligibility is preserved. - Does this need documentation: No, no new user setting. --- be/cmake/thirdparty.cmake | 6 +- be/src/exec/es/es_scan_reader.cpp | 9 +- be/src/exec/es/es_scan_reader.h | 1 + be/src/exec/es/es_scroll_parser.cpp | 308 +++++++++++------- be/src/exec/es/es_scroll_parser.h | 10 +- be/test/exec/es_scroll_parser_test.cpp | 96 +++++- .../doris/datasource/es/EsFieldContext.java | 6 + .../doris/datasource/es/EsMappingUtils.java | 52 ++- .../apache/doris/datasource/es/EsUtil.java | 3 +- .../datasource/es/source/EsScanNode.java | 10 +- .../datasource/es/EsMappingUtilsTest.java | 22 ++ .../source/EsScanNodeFileScannerV2Test.java | 39 +++ .../es/test_es_source_review_cases.groovy | 131 ++++++++ 13 files changed, 555 insertions(+), 138 deletions(-) create mode 100644 regression-test/suites/external_table_p2/es/test_es_source_review_cases.groovy diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 19b5c1ecc1dbd6..4f0d4d1578177f 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -189,10 +189,8 @@ if (OS_MACOSX) add_thirdparty(intl) endif() -# Only used on x86 or x86_64 -if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR "${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86_64") - add_thirdparty(deflate) -endif() +# The pinned ORC decoder uses libdeflate on ARM too; omitting it leaves unresolved decoder symbols. +add_thirdparty(deflate) add_thirdparty(icuuc LIB64) add_thirdparty(icui18n LIB64) diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp index f09538ba3e6e2b..0b807ab2241916 100644 --- a/be/src/exec/es/es_scan_reader.cpp +++ b/be/src/exec/es/es_scan_reader.cpp @@ -37,10 +37,10 @@ namespace doris { // hits.hits._id used for obtain ES document `_id` const std::string SOURCE_SCROLL_SEARCH_FILTER_PATH = - "filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id"; + "filter_path=_scroll_id,hits.hits._source,hits.hits._index,hits.total,hits.hits._id"; // hits.hits._score used for processing field not exists in one batch const std::string DOCVALUE_SCROLL_SEARCH_FILTER_PATH = - "filter_path=_scroll_id,hits.total,hits.hits._score,hits.hits.fields"; + "filter_path=_scroll_id,hits.total,hits.hits._index,hits.hits._score,hits.hits.fields"; const std::string REQUEST_PREFERENCE_PREFIX = "&preference=_shards:"; const std::string REQUEST_SEARCH_SCROLL_PATH = "/_search/scroll"; @@ -72,6 +72,10 @@ ESScanReader::ESScanReader(const std::string& target, std::istringstream(props.at(KEY_HTTP_SSL_ENABLED)) >> std::boolalpha >> _use_ssl_client; } + // The same numeric string can mean a compact calendar date or an epoch on different backings. + if (const auto formats = props.find("date_format_context"); formats != props.end()) { + _date_format_context = formats->second; + } std::string batch_size_str = props.at(KEY_BATCH_SIZE); _batch_size = atoi(batch_size_str.c_str()); std::string filter_path = @@ -216,6 +220,7 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr& scr } scroll_parser.reset(new ScrollParser(_doc_value_mode)); + RETURN_IF_ERROR(scroll_parser->set_date_format_context(_date_format_context)); VLOG_CRITICAL << "get_next request ES, returned response: " << response; Status status = scroll_parser->parse(response, _exactly_once); if (!status.ok()) { diff --git a/be/src/exec/es/es_scan_reader.h b/be/src/exec/es/es_scan_reader.h index a980ac1beba067..094f56f2d18bd6 100644 --- a/be/src/exec/es/es_scan_reader.h +++ b/be/src/exec/es/es_scan_reader.h @@ -71,6 +71,7 @@ class ESScanReader { std::string _type; // push down filter std::string _query; + std::string _date_format_context; // Elasticsearch shards to fetch document std::string _shards; // whether use ssl client diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index f70870f7e58bfb..1906e832e5cc24 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -428,106 +428,125 @@ static bool set_es_epoch_seconds(DateValue& value, int64_t seconds, return true; } -template -Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool is_date_str, - typename PrimitiveTypeTraits::CppType* slot, - const cctz::time_zone& time_zone) { - constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME; - typename PrimitiveTypeTraits::CppType dt_val; - if (is_date_str) { - std::string str_date(col.GetString(), col.GetStringLength()); - size_t str_length = col.GetStringLength(); - // The default strict_date_optional_time parser wins over epoch_millis for a four-digit year. - // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds after the epoch. - if (str_length == 4 && str_date.find_first_not_of("0123456789") == std::string::npos) { - str_date += "-01-01"; - str_length = str_date.size(); +static bool es_calendar_format_accepts(const std::string& text, std::string_view format, + bool* year_only) { + const bool digits = !text.empty() && text.find_first_not_of("0123456789") == std::string::npos; + *year_only = false; + // Compact dates win only for their mapped grammar. "20240101" is an epoch in the default + // format, whereas "2024" is a year only when an optional-date format precedes epoch_millis. + if (format == "basic_date" || format == "yyyyMMdd") { + return digits && text.size() == 8; + } + if (format == "strict_date_optional_time" || format == "date_optional_time" || + format == "strict_date_optional_time_nanos") { + if (digits) { + *year_only = text.size() == 4; + return *year_only; } - bool success = false; - int64_t epoch_seconds; - // Source fallback also reaches short, zero and fractional epoch strings, e.g. "0"/"1.5". - if (parse_es_epoch_millis(str_date, &epoch_seconds)) { - success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone); - } else if (str_length > 19) { - std::chrono::system_clock::time_point tp; - // time_zone suffix pattern - // Z/+08:00/-04:30 - RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); - bool ok = false; - std::string fmt; - re2::StringPiece value; - if (time_zone_pattern.Match(str_date, 0, str_date.size(), RE2::UNANCHORED, &value, 1)) { - // with time_zone info - // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00 - // or 2022-08-08T12:10:10.000Z or YYYY-MM-DDTHH:MM:SS-08:00 - fmt = "%Y-%m-%dT%H:%M:%E*S%Ez"; - cctz::time_zone ctz; - // find time_zone by time_zone suffix string - TimezoneUtils::find_cctz_time_zone(value.as_string(), ctz); - ok = cctz::parse(fmt, str_date, ctz, &tp); - } else { - // without time_zone info - // 2022-08-08T12:10:10.000 - fmt = "%Y-%m-%dT%H:%M:%E*S"; - // If the time without time_zone info, ES will assume it is UTC time. - // So we parse it in Doris with UTC time zone. - ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp); - } - if (ok) { - // The local time zone can change by session variable `time_zone` - // We should use the user specified time zone, not the actual system local time zone. - success = true; - // A pre-epoch fractional instant must floor to its containing second, like numeric dates. - dt_val.from_unixtime( - std::chrono::floor(tp).time_since_epoch().count(), - time_zone); - } - } else if (str_length == 19) { - // YYYY-MM-DDTHH:MM:SS - if (*(str_date.c_str() + 10) == 'T') { - std::chrono::system_clock::time_point tp; - const bool ok = - cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, cctz::utc_time_zone(), &tp); - if (ok) { - success = true; - dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone); - } - } else { - // YYYY-MM-DD HH:MM:SS - CastParameters params; - if constexpr (is_datetime_v1) { - success = CastToDateOrDatetime::from_string_non_strict_mode< - DatelikeTargetType::DATE_TIME>({str_date.c_str(), str_length}, dt_val, - nullptr, params); - } else if constexpr (T == TYPE_DATEV2) { - success = CastToDateV2::from_string_non_strict_mode( - {str_date.c_str(), str_length}, dt_val, nullptr, params); - } else { - success = CastToDatetimeV2::from_string_non_strict_mode( - {str_date.c_str(), str_length}, dt_val, nullptr, -1, params); - } - } + return true; + } + // Restrict named calendar shapes before trying a later epoch alternative. + if (format == "yyyy-MM-dd" || format == "strict_date" || format == "date") { + return text.size() == 10 && text[4] == '-' && text[7] == '-'; + } + if (format == "yyyy-MM-dd HH:mm:ss") { + return text.size() == 19 && text[4] == '-' && text[7] == '-' && text[10] == ' '; + } + return true; +} - } else { - // YYYY-MM-DD or others - CastParameters params; - if constexpr (is_datetime_v1) { - success = CastToDateOrDatetime::from_string_non_strict_mode< - DatelikeTargetType::DATE_TIME>({str_date.c_str(), str_length}, dt_val, - nullptr, params); - } else if constexpr (T == TYPE_DATEV2) { - success = CastToDateV2::from_string_non_strict_mode({str_date.c_str(), str_length}, - dt_val, nullptr, params); - } else { - success = CastToDatetimeV2::from_string_non_strict_mode( - {str_date.c_str(), str_length}, dt_val, nullptr, -1, params); +static bool parse_es_iso_datetime(const std::string& text, int64_t* seconds) { + std::chrono::system_clock::time_point time; + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + re2::StringPiece suffix; + bool success; + if (time_zone_pattern.Match(text, 0, text.size(), RE2::UNANCHORED, &suffix, 1)) { + cctz::time_zone zone; + TimezoneUtils::find_cctz_time_zone(suffix.as_string(), zone); + success = cctz::parse("%Y-%m-%dT%H:%M:%E*S%Ez", text, zone, &time); + } else { + // ES interprets an ISO timestamp without a suffix in UTC, not the session time zone. + success = cctz::parse("%Y-%m-%dT%H:%M:%E*S", text, cctz::utc_time_zone(), &time); + } + if (success) { + // A pre-epoch fractional instant floors to its containing second, as numeric dates do. + *seconds = std::chrono::floor(time).time_since_epoch().count(); + } + return success; +} + +template +static bool parse_es_calendar_date(const std::string& text, std::string_view format, + typename PrimitiveTypeTraits::CppType& value, + const cctz::time_zone& time_zone) { + bool year_only; + if (!es_calendar_format_accepts(text, format, &year_only)) { + return false; + } + if (year_only) { + return parse_es_calendar_date(text + "-01-01", format, value, time_zone); + } + if (text.size() > 19 || (text.size() == 19 && text[10] == 'T')) { + int64_t seconds; + // Validate the civil year before narrowing it, just as epoch inputs do. + return parse_es_iso_datetime(text, &seconds) && + set_es_epoch_seconds(value, seconds, time_zone); + } + CastParameters params; + if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) { + return CastToDateOrDatetime::from_string_non_strict_mode( + {text.data(), text.size()}, value, nullptr, params); + } else if constexpr (T == TYPE_DATEV2) { + return CastToDateV2::from_string_non_strict_mode({text.data(), text.size()}, value, nullptr, + params); + } else { + return CastToDatetimeV2::from_string_non_strict_mode({text.data(), text.size()}, value, + nullptr, -1, params); + } +} + +template +static bool parse_es_date_string(const std::string& text, std::string_view formats, + typename PrimitiveTypeTraits::CppType& value, + const cctz::time_zone& time_zone) { + if (formats.empty()) { + // Old FE requests carry no format context; preserve the default ES date grammar. + formats = "strict_date_optional_time||epoch_millis"; + } + while (!formats.empty()) { + const auto separator = formats.find("||"); + const auto format = formats.substr(0, separator); + if (format == "epoch_millis") { + int64_t seconds; + if (parse_es_epoch_millis(text, &seconds)) { + // Once a grammar accepts the input, an out-of-range date must fail rather than + // being reinterpreted by a later format with different semantics. + return set_es_epoch_seconds(value, seconds, time_zone); } + } else if (parse_es_calendar_date(text, format, value, time_zone)) { + return true; } + if (separator == std::string_view::npos) { + break; + } + formats.remove_prefix(separator + 2); + } + return false; +} - if (!success) { +template +Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool parse_date_text, + typename PrimitiveTypeTraits::CppType* slot, + const cctz::time_zone& time_zone, std::string_view date_format) { + constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME; + typename PrimitiveTypeTraits::CppType dt_val; + if (parse_date_text) { + const std::string text = col.IsString() + ? std::string(col.GetString(), col.GetStringLength()) + : json_value_to_string(col); + if (!parse_es_date_string(text, date_format, dt_val, time_zone)) { RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); } - } else { int64_t epoch_seconds; // Integral JSON values retain their exact bits; unsigned overflow cannot be a Doris date. @@ -565,13 +584,12 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool template Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_doc_value, typename PrimitiveTypeTraits::CppType* slot, - const cctz::time_zone& time_zone) { - // this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source + const cctz::time_zone& time_zone, std::string_view date_format) { if (col.IsNumber()) { - // ES process date/datetime field would use millisecond timestamp for index or docvalue - // processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms - // Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds - return get_date_value_int(col, type, false, slot, time_zone); + // ES applies the mapped grammar to source numeric tokens too. Numeric docvalues are + // canonical epoch values; requests from an old FE also retain their millisecond decoder. + const bool mapped_source = !pure_doc_value && !date_format.empty(); + return get_date_value_int(col, type, mapped_source, slot, time_zone, date_format); } else if (col.IsArray() && pure_doc_value && !col.Empty()) { // this would happened just only when `enable_docvalue_scan = true` // ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose @@ -579,22 +597,23 @@ Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_d // At present, we just process this string format date. After some PR were merged into Doris, we would impose `epoch_mills` for // date field's docvalue if (col[0].IsString()) { - return get_date_value_int(col[0], type, true, slot, time_zone); + return get_date_value_int(col[0], type, true, slot, time_zone, date_format); } // ES would return millisecond timestamp for date field, divided by 1000 because the unit for from_unixtime is seconds - return get_date_value_int(col[0], type, false, slot, time_zone); + return get_date_value_int(col[0], type, false, slot, time_zone, date_format); } else { // this would happened just only when `enable_docvalue_scan = false`, and field has string format date from _source RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true); RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); - return get_date_value_int(col, type, true, slot, time_zone); + return get_date_value_int(col, type, true, slot, time_zone, date_format); } } template Status fill_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_doc_value, - IColumn* col_ptr, const cctz::time_zone& time_zone) { + IColumn* col_ptr, const cctz::time_zone& time_zone, + std::string_view date_format) { typename PrimitiveTypeTraits::CppType data; - RETURN_IF_ERROR((get_date_int(col, type, pure_doc_value, &data, time_zone))); + RETURN_IF_ERROR((get_date_int(col, type, pure_doc_value, &data, time_zone, date_format))); col_ptr->insert_data(const_cast(reinterpret_cast(&data)), 0); return Status::OK(); } @@ -779,10 +798,12 @@ Status process_column(const rapidjson::Value& col, PrimitiveType sub_type, bool template Status process_date_column(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value, - Array& array, const cctz::time_zone& time_zone) { + Array& array, const cctz::time_zone& time_zone, + std::string_view date_format) { if (!col.IsArray()) { typename PrimitiveTypeTraits::CppType data; - RETURN_IF_ERROR((get_date_int(col, sub_type, pure_doc_value, &data, time_zone))); + RETURN_IF_ERROR( + (get_date_int(col, sub_type, pure_doc_value, &data, time_zone, date_format))); array.push_back(Field::create_field(data)); } else { for (const auto& sub_col : col.GetArray()) { @@ -792,7 +813,8 @@ Status process_date_column(const rapidjson::Value& col, PrimitiveType sub_type, continue; } typename PrimitiveTypeTraits::CppType data; - RETURN_IF_ERROR((get_date_int(sub_col, sub_type, pure_doc_value, &data, time_zone))); + RETURN_IF_ERROR((get_date_int(sub_col, sub_type, pure_doc_value, &data, time_zone, + date_format))); array.push_back(Field::create_field(data)); } } @@ -824,7 +846,7 @@ Status process_jsonb_column(const rapidjson::Value& col, PrimitiveType sub_type, Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value, Array& array, - const cctz::time_zone& time_zone) { + const cctz::time_zone& time_zone, std::string_view date_format) { switch (sub_type) { case TYPE_CHAR: case TYPE_VARCHAR: @@ -850,11 +872,12 @@ Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType sub // see https://github.com/apache/doris/pull/16304 // No need to support date and datetime types. case TYPE_DATEV2: { - return process_date_column(col, sub_type, pure_doc_value, array, time_zone); + return process_date_column(col, sub_type, pure_doc_value, array, time_zone, + date_format); } case TYPE_DATETIMEV2: { - return process_date_column(col, sub_type, pure_doc_value, array, - time_zone); + return process_date_column(col, sub_type, pure_doc_value, array, time_zone, + date_format); } case TYPE_JSONB: { return process_jsonb_column(col, sub_type, pure_doc_value, array); @@ -867,6 +890,49 @@ Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType sub ScrollParser::ScrollParser(bool doc_value_mode) : _size(0), _line_index(0) {} +Status ScrollParser::set_date_format_context(const std::string& context) { + _date_format_context.clear(); + if (context.empty()) { + return Status::OK(); + } + // The wire value is backing index -> field -> format. Reject malformed payloads before typed access. + rapidjson::Document formats; + formats.Parse(context.data(), context.size()); + if (formats.HasParseError() || !formats.IsObject()) { + return Status::InvalidArgument("Invalid ES date format context"); + } + for (const auto& index : formats.GetObject()) { + if (!index.value.IsObject()) { + return Status::InvalidArgument("Invalid ES backing-index date formats"); + } + auto& fields = _date_format_context[index.name.GetString()]; + for (const auto& field : index.value.GetObject()) { + if (!field.value.IsString()) { + return Status::InvalidArgument("Invalid ES field date format"); + } + fields.emplace(field.name.GetString(), + std::string(field.value.GetString(), field.value.GetStringLength())); + } + } + return Status::OK(); +} + +std::string_view ScrollParser::_date_format(const rapidjson::Value& hit, + const std::string& column) const { + if (!hit.HasMember("_index")) { + return {}; + } + const auto index = _date_format_context.find(hit["_index"].GetString()); + if (index == _date_format_context.end()) { + return {}; + } + const auto format = index->second.find(column); + if (format == index->second.end()) { + return {}; + } + return format->second; +} + ScrollParser::~ScrollParser() = default; Status ScrollParser::parse(const std::string& scroll_result, bool exactly_once) { @@ -997,6 +1063,7 @@ Status ScrollParser::fill_columns(const std::vector& slots, std::string details = absl::Substitute(INVALID_NULL_VALUE, col_name); return Status::RuntimeError(details); } + const auto date_format = [&] { return _date_format(obj, slot_desc->col_name()); }; switch (type) { case TYPE_CHAR: case TYPE_VARCHAR: @@ -1145,20 +1212,20 @@ Status ScrollParser::fill_columns(const std::vector& slots, } case TYPE_DATE: - RETURN_IF_ERROR( - fill_date_int(col, type, pure_doc_value, col_ptr, time_zone)); + RETURN_IF_ERROR(fill_date_int(col, type, pure_doc_value, col_ptr, time_zone, + date_format())); break; case TYPE_DATETIME: - RETURN_IF_ERROR( - fill_date_int(col, type, pure_doc_value, col_ptr, time_zone)); + RETURN_IF_ERROR(fill_date_int(col, type, pure_doc_value, col_ptr, + time_zone, date_format())); break; case TYPE_DATEV2: - RETURN_IF_ERROR( - fill_date_int(col, type, pure_doc_value, col_ptr, time_zone)); + RETURN_IF_ERROR(fill_date_int(col, type, pure_doc_value, col_ptr, + time_zone, date_format())); break; case TYPE_DATETIMEV2: { - RETURN_IF_ERROR( - fill_date_int(col, type, pure_doc_value, col_ptr, time_zone)); + RETURN_IF_ERROR(fill_date_int(col, type, pure_doc_value, col_ptr, + time_zone, date_format())); break; } case TYPE_ARRAY: { @@ -1167,7 +1234,8 @@ Status ScrollParser::fill_columns(const std::vector& slots, assert_cast(remove_nullable(slot_desc->type()).get()) ->get_nested_type() ->get_primitive_type(); - RETURN_IF_ERROR(parse_column(col, sub_type, pure_doc_value, array, time_zone)); + RETURN_IF_ERROR( + parse_column(col, sub_type, pure_doc_value, array, time_zone, date_format())); col_ptr->insert(Field::create_field(array)); break; } diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index e1f1106f624eb5..1460ca037a46d3 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "core/data_type/data_type.h" @@ -51,16 +52,23 @@ class ScrollParser { const cctz::time_zone& time_zone, std::span column_positions = {}); + Status set_date_format_context(const std::string& context); + const std::string& get_scroll_id(); int get_size() const; private: Status parse_column(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value, - Array& array, const cctz::time_zone& time_zone); + Array& array, const cctz::time_zone& time_zone, + std::string_view date_format); + std::string_view _date_format(const rapidjson::Value& hit, const std::string& column) const; std::string _scroll_id; int _size; rapidjson::SizeType _line_index; + // Ordered maps keep per-row format probes logarithmic for wide mappings. + std::map>, std::less<>> + _date_format_context; rapidjson::Document _document_node; rapidjson::Value _inner_hits_node; diff --git a/be/test/exec/es_scroll_parser_test.cpp b/be/test/exec/es_scroll_parser_test.cpp index 432c7e9923cb8c..0a86d4c8e7052a 100644 --- a/be/test/exec/es_scroll_parser_test.cpp +++ b/be/test/exec/es_scroll_parser_test.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "core/assert_cast.h" #include "core/column/column_array.h" @@ -60,13 +61,18 @@ class EsScrollParserTest : public testing::Test { static Status read_scalar(TPrimitiveType::type type, const std::string& json, MutableColumnPtr* column, bool array = false, - const cctz::time_zone& time_zone = cctz::utc_time_zone()) { + const cctz::time_zone& time_zone = cctz::utc_time_zone(), + const std::string& date_format = "") { auto descriptor = value_slot(type, array); descriptor.nullIndicatorBit = 0; SlotDescriptor slot(descriptor); ScrollParser parser(false); - RETURN_IF_ERROR( - parser.parse(R"({"hits":{"hits":[{"_source":{"value":)" + json + "}}]}}", true)); + if (!date_format.empty()) { + RETURN_IF_ERROR(parser.set_date_format_context("{\"test\":{\"value\":\"" + date_format + + "\"}}")); + } + RETURN_IF_ERROR(parser.parse( + R"({"hits":{"hits":[{"_index":"test","_source":{"value":)" + json + "}}]}}", true)); std::vector slots {&slot}; std::vector columns; columns.push_back(slot.type()->create_column()); @@ -81,6 +87,90 @@ class EsScrollParserTest : public testing::Test { } }; +TEST_F(EsScrollParserTest, DateStringsHonorMappedFormatOrder) { + for (auto type : {TPrimitiveType::DATE, TPrimitiveType::DATETIME, TPrimitiveType::DATEV2, + TPrimitiveType::DATETIMEV2}) { + SlotDescriptor slot(value_slot(type)); + for (const auto& [format, input, expected] : + std::vector> { + {"basic_date", "20240101", "2024-01-01 00:00:00"}, + {"yyyyMMdd", "20240101", "2024-01-01 00:00:00"}, + {"basic_date||epoch_millis", "20240101", "2024-01-01 00:00:00"}, + {"epoch_millis||basic_date", "20240101", "1970-01-01 05:37:20"}, + {"epoch_millis", "2024", "1970-01-01 00:00:02"}, + {"epoch_millis||strict_date_optional_time", "2024", "1970-01-01 00:00:02"}, + {"strict_date_optional_time||epoch_millis", "2024", "2024-01-01 00:00:00"}, + {"yyyy-MM-dd||epoch_millis", "20240101", "1970-01-01 05:37:20"}, + {"yyyy-MM-dd||epoch_millis", "2024-01-01", "2024-01-01 00:00:00"}}) { + SCOPED_TRACE(format + ":" + input); + std::vector tokens {"\"" + input + "\""}; + if (input.find_first_not_of("0123456789") == std::string::npos) { + tokens.push_back(input); + } + for (const auto& token : tokens) { + MutableColumnPtr column; + ASSERT_TRUE(read_scalar(type, token, &column, false, cctz::utc_time_zone(), format) + .ok()); + const bool date_only = + type == TPrimitiveType::DATE || type == TPrimitiveType::DATEV2; + EXPECT_EQ(date_only ? expected.substr(0, 10) : expected, + slot.type()->to_string( + assert_cast(*column).get_nested_column(), + 0)); + } + } + } +} + +TEST_F(EsScrollParserTest, DateFormatContextUsesEachHitsBackingIndex) { + SlotDescriptor slot(value_slot(TPrimitiveType::DATETIMEV2)); + ScrollParser parser(false); + ASSERT_TRUE( + parser.set_date_format_context( + R"({"calendar":{"value":"basic_date"},"epoch":{"value":"epoch_millis"}})") + .ok()); + ASSERT_TRUE(parser.parse(R"({"hits":{"hits":[ + {"_index":"calendar","_source":{"value":"20240101"}}, + {"_index":"epoch","_source":{"value":"20240101"}} + ]}})", + true) + .ok()); + std::vector slots {&slot}; + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + for (int row = 0; row < 2; ++row) { + ASSERT_TRUE(parser.fill_columns(slots, columns, &eof, {}, cctz::utc_time_zone()).ok()); + } + EXPECT_EQ("2024-01-01 00:00:00", slot.type()->to_string(*columns[0], 0)); + EXPECT_EQ("1970-01-01 05:37:20", slot.type()->to_string(*columns[0], 1)); +} + +TEST_F(EsScrollParserTest, NumericDocvaluesRemainEpochWithMappedCalendarFormats) { + SlotDescriptor slot(value_slot(TPrimitiveType::DATETIMEV2)); + ScrollParser parser(true); + ASSERT_TRUE(parser.set_date_format_context(R"({"test":{"value":"basic_date"}})").ok()); + ASSERT_TRUE( + parser.parse(R"({"hits":{"hits":[{"_index":"test","fields":{"value":[20240101]}}]}})", + true) + .ok()); + std::vector slots {&slot}; + std::vector columns; + columns.push_back(slot.type()->create_column()); + bool eof = false; + ASSERT_TRUE( + parser.fill_columns(slots, columns, &eof, {{"value", "value"}}, cctz::utc_time_zone()) + .ok()); + EXPECT_EQ("1970-01-01 05:37:20", slot.type()->to_string(*columns[0], 0)); +} + +TEST_F(EsScrollParserTest, RejectsMalformedDateFormatContexts) { + for (const auto* context : {"[", "[]", R"({"test":42})", R"({"test":{"value":42}})"}) { + ScrollParser parser(false); + EXPECT_FALSE(parser.set_date_format_context(context).ok()); + } +} + TEST_F(EsScrollParserTest, DateSourceEpochMillisStrings) { for (auto type : {TPrimitiveType::DATE, TPrimitiveType::DATETIME, TPrimitiveType::DATEV2, TPrimitiveType::DATETIMEV2}) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java index 04601f47f34aa7..27bf04d7a80c92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsFieldContext.java @@ -40,6 +40,7 @@ public class EsFieldContext { private final Map fetchFieldsContext = new HashMap<>(); private final Map docValueFieldsContext = new HashMap<>(); + private final Map> dateFormatContext = new HashMap<>(); private final List needCompatDateFields = new ArrayList<>(); private final Set sourceValueUnsafeFields = new HashSet<>(); private final Set sourceModeUnsafeFields = new HashSet<>(); @@ -55,6 +56,11 @@ public Map getDocValueFieldsContext() { return docValueFieldsContext; } + /** Backing index -> column -> ordered date formats; aliases can have different grammars. */ + public Map> getDateFormatContext() { + return dateFormatContext; + } + public List getNeedCompatDateFields() { return needCompatDateFields; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java index 74454353757782..ae098c31cd6a05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMappingUtils.java @@ -24,9 +24,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -50,6 +52,9 @@ public final class EsMappingUtils { "byte", "short", "integer", "long", "unsigned_long", "float", "double", "half_float", "scaled_float", "boolean", "date", "date_nanos", "ip", "geo_point", "geo_shape")); + private static final Set SOURCE_NUMERIC_TYPES = new HashSet<>(Arrays.asList( + "byte", "short", "integer", "long", "unsigned_long", "float", "double", "half_float", "scaled_float")); + private EsMappingUtils() { } @@ -57,17 +62,32 @@ private EsMappingUtils() { * Parse the top-level "mappings" node from an index mapping JSON response. */ public static ObjectNode getMapping(String indexMapping) { + return getMapping(indexMapping, null); + } + + public static ObjectNode getMapping(String indexMapping, String mappingType) { try { ObjectNode jsonNodes = (ObjectNode) MAPPER.readTree(indexMapping); + ObjectNode fallback = null; // Alias mapping response order is unspecified. A bare/typed empty backing can appear // first; select a populated schema without discarding its _meta array annotations. for (JsonNode index : jsonNodes) { ObjectNode mapping = (ObjectNode) index.get("mappings"); - ObjectNode root = getRootSchema(mapping.deepCopy(), null, new ArrayList<>(), true); + ObjectNode root = getRootSchema(mapping.deepCopy(), mappingType, new ArrayList<>(), true); if (root.path("properties").size() > 0) { - return mapping; + // An ES5 alias can expose several types. Prefer the requested type even when + // another backing's unrelated type appears first in the mapping response. + if (mappingType == null || mapping.has(mappingType)) { + return mapping; + } + if (fallback == null) { + fallback = mapping; + } } } + if (fallback != null) { + return fallback; + } // Preserve the existing diagnostic when no backing index provides a usable schema. return (ObjectNode) jsonNodes.iterator().next().get("mappings"); } catch (IOException e) { @@ -133,7 +153,7 @@ private static boolean allValuesAreObjects(JsonNode node) { */ public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, String mappingType) { - ObjectNode mappings = getMapping(indexMapping); + ObjectNode mappings = getMapping(indexMapping, mappingType); ObjectNode rootSchema = getRootSchema(mappings, mappingType, new ArrayList<>()); ObjectNode properties = (ObjectNode) rootSchema.get("properties"); if (properties == null) { @@ -212,11 +232,31 @@ private static void resolveSourceValueSafety(EsFieldContext context, String inde resolveKeywordFields(context, field, column, field.path("type").asText()); } Set unsafeDocValues = new HashSet<>(); - for (JsonNode index : indices) { + Iterator> backings = indices.fields(); + while (backings.hasNext()) { + Map.Entry backing = backings.next(); + JsonNode index = backing.getValue(); ObjectNode root = getRootSchema((ObjectNode) index.get("mappings"), mappingType, new ArrayList<>(), true); JsonNode mappingDefaults = index.path("settings").path("index").path("mapping"); JsonNode properties = root.path("properties"); + Map dateFormats = new HashMap<>(); + for (String column : columns) { + JsonNode field = properties.path(column); + if ("alias".equals(field.path("type").asText())) { + field = properties.path(field.path("path").asText()); + } + if ("date".equals(field.path("type").asText()) + || "date_nanos".equals(field.path("type").asText())) { + // Preserve the mapping's order: basic_date and epoch_millis interpret the + // same numeric text differently. Each alias backing keeps its own formats. + dateFormats.put(column, field.path("format") + .asText("strict_date_optional_time||epoch_millis")); + } + } + if (!dateFormats.isEmpty()) { + context.getDateFormatContext().put(backing.getKey(), dateFormats); + } if (isSourceDisabled(root.path("_source"))) { // A source projection fails on disabled backings; full source preserves the // enabled backings of a mixed alias while disabled hits remain NULL. @@ -278,9 +318,11 @@ private static boolean changesSourceValue(JsonNode field, JsonNode mappingDefaul private static boolean requiresSourcePredicateEvaluation(JsonNode field) { String type = field.path("type").asText(); + // Source numeric tokens can round differently from ES indexing (e.g. 0.9999999999999999 + // becomes integer 1 after JSON decoding). Keep their predicates local only in source mode. // Source dates use the session time zone, including epoch_millis in explicit formats, // so their local value can differ from the UTC value used by an ES predicate. - return "scaled_float".equals(type) || "half_float".equals(type) || "alias".equals(type) + return SOURCE_NUMERIC_TYPES.contains(type) || "alias".equals(type) || field.hasNonNull("script") || "date".equals(type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java index 8682a4f4526df4..25c9977033908a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java @@ -112,7 +112,8 @@ public static ObjectNode getMappingProps(String sourceIndex, String indexMapping public static List genColumnsFromEs(EsRestClient client, String indexName, String mappingType, boolean mappingEsId, Map column2typeMap) { String mapping = client.getMapping(indexName); - ObjectNode mappings = getMapping(mapping); + // Inference and scans must select the same legacy type across alias backings. + ObjectNode mappings = EsMappingUtils.getMapping(mapping, mappingType); // Get array_fields while removing _meta property. List arrayFields = new ArrayList<>(); ObjectNode rootSchema = getRootSchema(mappings, mappingType, arrayFields); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index c486dd9ab77d93..1b640879d7da55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -41,6 +41,7 @@ import org.apache.doris.datasource.es.QueryBuilders.BoolQueryBuilder; import org.apache.doris.datasource.es.QueryBuilders.BuilderOptions; import org.apache.doris.datasource.es.QueryBuilders.QueryBuilder; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.PartitionPruner; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.RangePartitionPrunerV2; @@ -200,6 +201,9 @@ private TEsScanNode buildEsScanNode() { properties.put("read_full_source", "true"); } properties.put(EsResource.QUERY_DSL, queryBuilder.toJson()); + if (!table.fieldContext().getDateFormatContext().isEmpty()) { + properties.put("date_format_context", GsonUtils.GSON.toJson(table.fieldContext().getDateFormatContext())); + } if (table.isEnableKeywordSniff() && table.fieldsContext().size() > 0) { esScanNode.setFieldsContext(table.fieldsContext()); } @@ -507,8 +511,10 @@ private void collectSourceUnsafeFields(Expr expr, Set fields) { String name = ((SlotRef) expr).getColumnName(); if (!"_id".equals(name) && table.getColumn(name) != null && (table.getColumn(name).getType().isStringType() - || table.getColumn(name).getType().isArrayType())) { - // A source string can be the whole JSON array instead of one indexed element. + || table.getColumn(name).getType().isArrayType() + || table.getColumn(name).getType().isJsonbType())) { + // Source containers such as {} are nonnull even when ES indexes no leaf value. + // A source string can also be the whole JSON array instead of one indexed element. fields.add(name); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java index 1c378e5a7033b4..9257381f373ab5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/EsMappingUtilsTest.java @@ -197,6 +197,28 @@ public void legacyTypeNamesCanResembleRootParameters() { } } + @Test + void aliasBackingSelectionHonorsTheRequestedLegacyType() { + String mapping = "{\"a\":{\"mappings\":{\"empty\":{},\"wanted\":{\"properties\":" + + "{\"x\":{\"type\":\"long\"}}}}},\"b\":{\"mappings\":{\"other\":{\"properties\":" + + "{\"y\":{\"type\":\"long\"}}}}}}"; + Assertions.assertEquals("long", EsMappingUtils.getMappingProps("alias", mapping, "wanted") + .path("x").path("type").asText()); + Assertions.assertEquals("long", EsMappingUtils.resolveFieldContext( + Collections.singletonList("x"), "alias", mapping, "wanted").getColumn2typeMap().get("x")); + } + + @Test + void sourceNumericPredicatesStayLocalWithoutDisablingDocvalues() { + for (String type : Arrays.asList("byte", "short", "integer", "long", "unsigned_long", "float", "double")) { + String mapping = "{\"test\":{\"mappings\":{\"properties\":{\"n\":{\"type\":\"" + type + "\"}}}}}"; + EsFieldContext fields = EsMappingUtils.resolveFieldContext( + Collections.singletonList("n"), "test", mapping, null); + Assertions.assertTrue(fields.getSourceModeUnsafeFields().contains("n"), type); + Assertions.assertEquals("n", fields.getDocValueFieldsContext().get("n"), type); + } + } + @Test void runtimeFieldNamedPropertiesIsNotALegacyType() throws Exception { ObjectNode mapping = (ObjectNode) new ObjectMapper().readTree( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java index dd8cec09e415fd..a7b195d2afe60a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/es/source/EsScanNodeFileScannerV2Test.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; @@ -35,6 +36,7 @@ import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.es.DorisEsException; import org.apache.doris.datasource.es.EsFieldContext; +import org.apache.doris.datasource.es.EsMappingUtils; import org.apache.doris.datasource.es.EsShardPartitions; import org.apache.doris.datasource.es.EsShardRouting; import org.apache.doris.datasource.es.EsTablePartitions; @@ -193,6 +195,43 @@ void sourceStringsAndNativeResidualsAreHandledBeforePushdown() throws Exception } } + @Test + void mappedDateFormatsReachBothScanProtocols() throws Exception { + String mapping = "{\"test\":{\"mappings\":{\"properties\":{\"id\":" + + "{\"type\":\"date\",\"format\":\"basic_date||epoch_millis\"}}}}}"; + for (boolean v2 : Arrays.asList(false, true)) { + TestScanNode node = createNode(v2); + EsTable table = (EsTable) node.getTupleDesc().getTable(); + Mockito.doReturn(EsMappingUtils.resolveFieldContext(Collections.singletonList("id"), + "test", mapping, null)).when(table).fieldContext(); + java.util.Map properties = v2 + ? node.getScanRangeLocations(0).get(0).getScanRange().getExtScanRange() + .getFileScanRange().getParams().getEsProperties() + : node.serialize().getEsScanNode().getProperties(); + Assertions.assertEquals("{\"test\":{\"id\":\"basic_date||epoch_millis\"}}", + properties.get("date_format_context")); + } + } + + @Test + void sourceJsonbNullPredicatesStayLocalWithBothScanners() throws Exception { + for (boolean v2 : Arrays.asList(false, true)) { + for (boolean notNull : Arrays.asList(false, true)) { + TestScanNode node = createNode(v2); + EsTable table = (EsTable) node.getTupleDesc().getTable(); + Mockito.doReturn(new Column("obj", Type.JSONB)).when(table).getColumn("obj"); + Expr predicate = new IsNullPredicate(new SlotRef(null, "obj"), notNull); + node.getConjuncts().add(predicate); + Method build = EsScanNode.class.getDeclaredMethod("buildQuery"); + build.setAccessible(true); + build.invoke(node); + Assertions.assertEquals(Collections.singletonList(predicate), node.getConjuncts()); + Assertions.assertTrue(node.getNodeExplainString("", org.apache.doris.thrift.TExplainLevel.NORMAL) + .contains("REMOTE_PREDICATES: {\"match_all\":{}}")); + } + } + } + @Test void finalReadModeControlsPredicateSafetyAndFullSourceRequest() throws Exception { for (boolean docvalues : Arrays.asList(false, true)) { diff --git a/regression-test/suites/external_table_p2/es/test_es_source_review_cases.groovy b/regression-test/suites/external_table_p2/es/test_es_source_review_cases.groovy new file mode 100644 index 00000000000000..6063c82239d97f --- /dev/null +++ b/regression-test/suites/external_table_p2/es/test_es_source_review_cases.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +suite("test_es_source_review_cases", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("es_8_port") + assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured") + String endpoint = "http://${host}:${port}" + def request = { String method, String path, Object body = null -> + def connection = new URL("${endpoint}/${path}").openConnection() + connection.requestMethod = method + connection.connectTimeout = 10000 + connection.readTimeout = 30000 + try { + if (body != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", "application/json") + connection.outputStream.withCloseable { it.write(JsonOutput.toJson(body).getBytes("UTF-8")) } + } + int status = connection.responseCode + if (method == "DELETE" && status == 404) { + return null + } + String text = (status < 400 ? connection.inputStream : connection.errorStream).getText("UTF-8") + assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: ${status} ${text}") + return new JsonSlurper().parseText(text) + } finally { + connection.disconnect() + } + } + request("DELETE", "doris_source_review_cases") + request("PUT", "doris_source_review_cases", [ + settings: [number_of_shards: 1, number_of_replicas: 0, + analysis: [normalizer: [folded: [type: "custom", filter: ["lowercase"]]]]], + mappings: [properties: [row_id: [type: "long"], tag: [type: "keyword", normalizer: "folded"], + n: [type: "integer"], f: [type: "float"], obj: [type: "flattened"], + epoch: [type: "date", format: "epoch_millis"], basic: [type: "date", format: "basic_date"]]]]) + [[row_id: 1, tag: "UPPER", n: new BigDecimal("0.9999999999999999"), + f: new BigDecimal("1.0000000596046448"), obj: [:], epoch: "2024", basic: "20240101"], + [row_id: 2, tag: "UPPER", n: null, f: null, obj: null, epoch: 2024, basic: 20240101], + [row_id: 3, tag: "UPPER", n: 1, f: 1.0, obj: [key: "value"], epoch: "0", basic: "20240101"]].each { row -> + request("PUT", "doris_source_review_cases/_doc/${row.row_id}", row) + } + request("POST", "doris_source_review_cases/_refresh") + + sql "switch internal" + sql "drop table if exists es_source_review_control" + sql """create table es_source_review_control (row_id bigint, tag string, n int, f float, obj json) + distributed by hash(row_id) buckets 1 properties("replication_num"="1")""" + String control = "internal.${context.dbName}.es_source_review_control" + sql "drop table if exists es_source_review_dates" + sql """create table es_source_review_dates (row_id bigint, tag string, basic datetimev2(0)) + engine=elasticsearch properties("hosts"="${endpoint}", "index"="doris_source_review_cases", + "nodes_discovery"="false", "enable_docvalue_scan"="false")""" + String legacyDates = "internal.${context.dbName}.es_source_review_dates" + // Legacy ES table metadata is populated by the repository daemon, not by CREATE TABLE. + awaitUntil(30, 1) { + try { + sql "explain select row_id,tag,basic from ${legacyDates}" + return true + } catch (Exception error) { + if (error.message.contains("EsTable metadata has not been synced")) { + return false + } + throw error + } + } + sql "drop catalog if exists es_source_review_catalog" + sql """create catalog es_source_review_catalog properties("type"="es", "hosts"="${endpoint}", + "nodes_discovery"="false", "enable_docvalue_scan"="true")""" + def originalScanner = sql("select @@enable_file_scanner_v2")[0][0] + def originalParallel = sql("show variables like 'enable_es_parallel_scroll'")[0][1] + def originalZone = sql("select @@time_zone")[0][0] + try { + sql "set time_zone = 'UTC'" + sql "switch es_source_review_catalog" + // Compare filters with the values Doris actually materializes, independently of ES's + // numeric indexing conversion. The normalized tag makes this entire projection use source. + sql """insert into ${control} select row_id,tag,n,f,obj from doris_source_review_cases""" + [false, true].each { scanner -> + sql "set enable_file_scanner_v2 = ${scanner}" + [false, true].each { parallel -> + sql "set enable_es_parallel_scroll = ${parallel}" + String tag = "v2_${scanner}_parallel_${parallel}" + ["n = 1", "n != 1", "n > 0", "n is null", "n = 1 or row_id = 3", "not (n = 1)", + "f = cast(1 as float)", "f != cast(1 as float)", "obj is null", "obj is not null"].eachWithIndex { + String predicate, int i -> + quickTest("${tag}_filter_${i}", + "select row_id,tag,n,f from doris_source_review_cases where ${predicate} order by row_id", + "select row_id,tag,n,f from ${control} where ${predicate} order by row_id") + } + quickTest("${tag}_local_filter_limit", + "select row_id,tag from doris_source_review_cases where n = 1 order by row_id limit 1", + "select row_id,tag from ${control} where n = 1 order by row_id limit 1") + // Explicit legacy DATETIMEV2 columns must honor basic_date for strings and numeric tokens. + quickTest("${tag}_legacy_basic_date", + "select row_id,tag,basic from ${legacyDates} order by row_id", + "select row_id,tag,cast('2024-01-01' as datetimev2(0)) from ${control} order by row_id") + // epoch_millis-only accepts both the string and numeric 2024 as milliseconds, not a year. + quickTest("${tag}_epoch_format", + "select row_id,tag,cast(epoch as date) from doris_source_review_cases order by row_id", + "select row_id,tag,cast('1970-01-01' as date) from ${control} order by row_id") + } + } + } finally { + sql "set enable_file_scanner_v2 = ${originalScanner}" + sql "set enable_es_parallel_scroll = ${originalParallel}" + sql "set time_zone = '${originalZone}'" + sql "switch internal" + } +}