From a066846e1bc4a3bfb8747aa2c8ef38eeea268ab3 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sat, 10 Oct 2026 18:20:11 +0800 Subject: [PATCH 1/5] [feature](variant) Support relational operations for Variant V2 on branch-4.2 ### What problem does this PR solve? Related PR: #67675 Problem Summary: Backport Variant V2 canonical equality, sorting and relational operations, runtime-filter guards and JSON function support to branch-4.2. Preserve the default disabled V2 configuration, legacy Variant restrictions, target serialization APIs and Variant properties. Do not restore schema-template auto cast. Restrict TIMESTAMPTZ paths in newly defined Variant schema templates. ### Release note When Variant V2 is enabled, Variant values support equality joins and ordering, and can be passed to JSON functions. Variant V2 remains disabled by default. ### Check List (For Author) - Test: ASAN BE/FE builds; 114 related BE unit tests; 86 FE unit tests; 25 regression suites passed. - Expanded BE run: 858 passed, 5 skipped, 1 unchanged V1 byte_size golden mismatch; reproduced alone. - clang-format 16 and FE Checkstyle passed. clang-tidy blocked by existing NOLINTEND and analyzer crash. - Behavior changed: Yes, enable relational and JSON operations for Variant V2. - Does this need documentation: No, backport of #67675. (cherry picked from commit cd1508ca6e05650126dcf99ebdd19818f2be626b) --- .../column/variant_v2/column_variant_v2.cpp | 205 +++++++++++++- .../column/variant_v2/column_variant_v2.h | 27 ++ .../core/data_type/data_type_variant_v2.cpp | 8 +- .../data_type_serde/data_type_array_serde.cpp | 5 +- .../data_type_serde/data_type_array_serde.h | 3 +- .../data_type_decimal_serde.cpp | 13 +- .../data_type_serde/data_type_decimal_serde.h | 8 +- .../data_type_serde/data_type_jsonb_serde.cpp | 3 +- .../data_type_serde/data_type_jsonb_serde.h | 3 +- .../data_type_serde/data_type_map_serde.cpp | 5 +- .../data_type_serde/data_type_map_serde.h | 3 +- .../data_type_nullable_serde.cpp | 5 +- .../data_type_nullable_serde.h | 3 +- .../data_type_number_serde.cpp | 13 +- .../data_type_serde/data_type_number_serde.h | 8 +- .../core/data_type_serde/data_type_serde.cpp | 11 +- be/src/core/data_type_serde/data_type_serde.h | 9 +- .../data_type_string_serde.cpp | 6 +- .../data_type_serde/data_type_string_serde.h | 3 +- .../data_type_struct_serde.cpp | 5 +- .../data_type_serde/data_type_struct_serde.h | 3 +- .../data_type_variant_v2_serde.cpp | 43 +++ .../data_type_variant_v2_serde.h | 7 + .../core/value/variant/variant_canonical.cpp | 262 +++++++++++++++--- be/src/core/value/variant/variant_canonical.h | 25 ++ be/src/exec/sort/sort_block.cpp | 8 + be/src/exec/sort/sort_block.h | 6 +- be/src/exprs/function/cast/cast_to_jsonb.h | 11 +- .../variant_v2/cast_variant_to_scalar.cpp | 7 +- be/src/exprs/function/function_to_json.cpp | 9 +- .../variant/v2/variant_path_builder.cpp | 4 +- .../java/org/apache/doris/catalog/Type.java | 3 +- .../post/RuntimeFilterPushDownVisitor.java | 7 +- .../rules/analysis/CheckAfterRewrite.java | 29 +- .../rules/rewrite/EliminateOrderByKey.java | 6 +- .../ExplicitlyCastableSignature.java | 7 - .../functions/scalar/JsonHash.java | 4 +- .../expressions/functions/scalar/ToJson.java | 4 +- .../doris/nereids/util/TypeCoercionUtils.java | 35 ++- 39 files changed, 697 insertions(+), 129 deletions(-) diff --git a/be/src/core/column/variant_v2/column_variant_v2.cpp b/be/src/core/column/variant_v2/column_variant_v2.cpp index 7d7317d1e0ca7c..70fc0d25c49619 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.cpp +++ b/be/src/core/column/variant_v2/column_variant_v2.cpp @@ -45,6 +45,7 @@ #include "core/value/variant/variant_canonical.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/sort/sort_block.h" namespace doris { namespace { @@ -689,6 +690,33 @@ const IColumn& ColumnVariantV2::typed_column() const { return *_typed; } +void ColumnVariantV2::validate_typed_rows() const { + DORIS_CHECK(_typed != nullptr) << "validate_typed_rows requires ColumnVariantV2 typed state"; + switch (_typed_type->get_primitive_type()) { + // Every value of these types maps to a Variant scalar, so there is nothing to reject and no + // reason to format each LARGEINT or IP address only to discard the text. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_IPV4: + case TYPE_IPV6: + return; + default: + break; + } + // with_variant_typed_scalar() is the one definition of what Variant can encode: it throws for + // an invalid date, a string that is not UTF-8 or a decimal outside its width. A type missing + // from the list above is validated, which is the safe side. + visit_typed_scalar_column(assert_cast(*_typed), + _typed_type->get_primitive_type(), _typed_type->get_scale(), 0, + _typed->size(), [](size_t, const VariantScalarRef&) {}); +} + const DataTypePtr& ColumnVariantV2::typed_type() const { DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state"; return _typed_type; @@ -1285,6 +1313,132 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row) const { return {.metadata = {.data = metadata.data, .size = metadata.size}, .value = value}; } +std::pair ColumnVariantV2::_typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const { + if (!is_typed() || !right.is_typed() || + !(_typed_type == right._typed_type || _typed_type->equals(*right._typed_type))) { + return {nullptr, nullptr}; + } + // Admission is a whitelist: a typed type takes the native path only after its native order has + // been checked against the canonical Variant order (scalar_compare in variant_canonical.cpp). + // The premises shared by every admitted type: + // - both sides have the same typed type including its parameters, so DECIMAL scales and + // DATETIME precisions match and native values are directly comparable; + // - every non-null typed row is a value Variant can encode (the rule and who enforces it + // are at create_typed()), so the native order does not rank a value the canonical path + // would reject, such as an invalid date or a string that is not UTF-8; + // - a typed NULL is the Variant null, which the callers order first, as the canonical order + // does. + // A type that is not listed falls back to the canonical comparison, which is always correct. + switch (_typed_type->get_primitive_type()) { + // Exact numerics and temporal values: both orders are the numeric/chronological order. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DATE: + case TYPE_DATEV2: + case TYPE_DATETIME: + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMPTZ: + // Strings: both orders compare the bytes as unsigned. + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + break; + // Canonical order puts every NaN last and equal to each other, and -0 equal to +0. compare_at() + // of a floating point column orders the same way; its compare_internal() uses < and >, which + // leave NaN unordered. + case TYPE_FLOAT: + case TYPE_DOUBLE: + if (!allow_floating) { + return {nullptr, nullptr}; + } + break; + // Not admitted: LARGEINT magnitudes above 10^38 - 1 are Variant strings, and IPv4/IPv6 are + // Variant strings whose byte order is not the address order. + default: + return {nullptr, nullptr}; + } + return {&assert_cast(typed_column()), + &assert_cast(right.typed_column())}; +} + +int ColumnVariantV2::compare_at(size_t n, size_t m, const IColumn& rhs, + int nan_direction_hint) const { + const auto& right = assert_cast(rhs); + DCHECK_LT(n, size()); + DCHECK_LT(m, right.size()); + + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, true); + left_nullable != nullptr) { + if (left_nullable->is_null_at(n)) { + return right_nullable->is_null_at(m) ? 0 : -1; + } + if (right_nullable->is_null_at(m)) { + return 1; + } + return left_nullable->get_nested_column().compare_at( + n, m, right_nullable->get_nested_column(), nan_direction_hint); + } + + // A column is in exactly one of three states. Encoded rows are canonical Variant bytes that + // get_value_ref() reads in place. A shredded column has no row bytes until its state + // materializes them; get_value_ref() asks the state for them, which builds and caches them on + // the first call. A typed row is a scalar, compared below without being encoded: a sort compares + // O(n log n) times, so nothing here may allocate or encode per comparison. + if (!is_typed() && !right.is_typed()) { + return canonical_compare(get_value_ref(n), right.get_value_ref(m)); + } + if (!is_typed()) { + // Keep the typed side on the left; canonical_compare() is antisymmetric. + return -right.compare_at(m, n, *this, nan_direction_hint); + } + // The typed fast path declined (different typed types, a type outside its whitelist, or an + // encoded/shredded right side). A typed null row is the canonical null scalar, which sorts + // first - the same order the typed fast path applies. + int result = 0; + const auto& left_nullable = + assert_cast(typed_column()); + visit_typed_scalar_column( + left_nullable, _typed_type->get_primitive_type(), _typed_type->get_scale(), n, n + 1, + [&](size_t, const VariantScalarRef& left_scalar) { + if (!right.is_typed()) { + result = canonical_compare(left_scalar, right.get_value_ref(m)); + return; + } + visit_typed_scalar_column( + assert_cast( + right.typed_column()), + right._typed_type->get_primitive_type(), right._typed_type->get_scale(), m, + m + 1, [&](size_t, const VariantScalarRef& right_scalar) { + result = canonical_compare(left_scalar, right_scalar); + }); + }); + return result; +} + +void ColumnVariantV2::compare_internal(size_t rhs_row_id, const IColumn& rhs, + int nan_direction_hint, int direction, + std::vector& cmp_res, + uint8_t* __restrict filter) const { + const auto& right = assert_cast(rhs); + // ColumnVector::compare_internal does not provide Variant's canonical NaN ordering, so floating + // point typed columns take the canonical path below. + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, false); + left_nullable != nullptr) { + left_nullable->compare_internal(rhs_row_id, *right_nullable, -1, direction, cmp_res, + filter); + return; + } + IColumn::compare_internal(rhs_row_id, rhs, nan_direction_hint, direction, cmp_res, filter); +} + Field ColumnVariantV2::operator[](size_t row) const { Field result; get(row, result); @@ -1662,6 +1816,29 @@ void ColumnVariantV2::pop_back(size_t length) { _check_invariants(); } +void ColumnVariantV2::erase(size_t start, size_t length) { + DORIS_CHECK_LE(start, size()) << "erase start exceeds the column size"; + DORIS_CHECK_LE(length, size() - start) << "erase range exceeds the column size"; + if (length == 0) { + return; + } + if (_shredded) { + ensure_encoded(); + } + if (_typed) { + mutate_subcolumn(_typed); + _typed->erase(start, length); + _check_invariants(); + return; + } + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + // The metadata dictionary keeps blobs that no remaining row references; ids stay valid. + _values->erase(start, length); + _meta_ids->erase(start, length); + _check_invariants(); +} + StringRef ColumnVariantV2::get_data_at(size_t) const { throw_unsupported("get_data_at"); } @@ -2139,8 +2316,32 @@ void ColumnVariantV2::resize(size_t new_size) { } } -void ColumnVariantV2::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const { - throw_unsupported("get_permutation"); +void ColumnVariantV2::get_permutation(bool reverse, size_t limit, int nan_direction_hint, + HybridSorter& sorter, Permutation& result) const { + const size_t row_count = size(); + result.resize(row_count); + for (size_t row = 0; row < row_count; ++row) { + result[row] = row; + } + + const auto less = [&](size_t left, size_t right) { + const int comparison = compare_at(left, right, *this, nan_direction_hint); + return reverse ? comparison > 0 : comparison < 0; + }; + // std::partial_sort only pays off when limit << row_count, the same threshold ColumnVector uses. + if (static_cast(limit) > static_cast(row_count) / 8.0) { + limit = 0; + } + if (limit != 0 && limit < row_count) { + std::partial_sort(result.begin(), result.begin() + limit, result.end(), less); + } else { + sorter.sort(result.begin(), result.end(), less); + } +} + +void ColumnVariantV2::sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const { + sorter->sort_column(*this, flags, perms, range, last_column); } void ColumnVariantV2::replace_column_data(const IColumn&, size_t, size_t) { diff --git a/be/src/core/column/variant_v2/column_variant_v2.h b/be/src/core/column/variant_v2/column_variant_v2.h index affea6a7e34b7a..d096abd23fa8d8 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.h +++ b/be/src/core/column/variant_v2/column_variant_v2.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "common/exception.h" #include "core/assert_cast.h" @@ -179,7 +180,21 @@ class ColumnVariantV2 final : public COWHelper { // The input must be an exact, non-Const ColumnNullable whose nested column matches the // non-nullable supported scalar type. + // + // Every row that is non-null in the typed column's own null map must be a value Variant can + // encode. That null map is the only mask every consumer honours: hashing, for one, maps a row + // to a scalar before it looks at an outer null map. Nothing is checked here, because validity + // is enforced where a row becomes a scalar (with_variant_typed_scalar() throws), which covers + // hashing, serialization, encoding and the canonical comparison. The native comparison paths + // do not go through that mapping and rely on the rule instead: + // - producers that decode stored or transported Variant data build valid rows; an external + // Parquet file is trusted to follow its specification, and a violation surfaces as an + // error when the row is hashed or encoded; + // - CAST from a scalar takes arbitrary values, so it folds its null map into the typed one + // and calls validate_typed_rows(). static MutablePtr create_typed(ColumnPtr column, DataTypePtr scalar_type); + // Throws unless every non-null typed row is a value Variant can encode. + void validate_typed_rows() const; static MutablePtr create_shredded(std::shared_ptr state); bool is_typed() const noexcept { return _typed != nullptr; } @@ -224,6 +239,11 @@ class ColumnVariantV2 final : public COWHelper { // including insert, clear, COW mutation, or future row transformations, may invalidate it. VariantRef get_value_ref(size_t row) const; + int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override; + void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint, + int direction, std::vector& cmp_res, + uint8_t* __restrict filter) const override; + Field operator[](size_t row) const override; void get(size_t row, Field& result) const override; void insert(const Field& field) override; @@ -235,6 +255,8 @@ class ColumnVariantV2 final : public COWHelper { void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, const uint32_t* indices_end) override; void pop_back(size_t length) override; + // The analytic sink erases the rows it has consumed from its partition and order key columns. + void erase(size_t start, size_t length) override; StringRef get_data_at(size_t row) const override; void insert_data(const char* pos, size_t length) override; @@ -272,6 +294,8 @@ class ColumnVariantV2 final : public COWHelper { void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter, Permutation& result) const override; + void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const override; void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override; private: @@ -289,6 +313,9 @@ class ColumnVariantV2 final : public COWHelper { void _adopt_state_from(ColumnVariantV2& replacement); void _detach_metadata_for_write(); void _check_invariants() const; + std::pair _typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const; + static void mutate_subcolumn(IColumn::WrappedPtr& subcolumn) { static_cast(subcolumn) = std::move(*static_cast(subcolumn)).mutate(); diff --git a/be/src/core/data_type/data_type_variant_v2.cpp b/be/src/core/data_type/data_type_variant_v2.cpp index 92b0fb4cbd209b..456bc3884483d9 100644 --- a/be/src/core/data_type/data_type_variant_v2.cpp +++ b/be/src/core/data_type/data_type_variant_v2.cpp @@ -48,10 +48,10 @@ MutableColumnPtr DataTypeVariantV2::create_column() const { } bool DataTypeVariantV2::equals(const IDataType& rhs) const { - const auto* rhs_type = typeid_cast(&rhs); - return rhs_type != nullptr && - _max_subcolumns_count == rhs_type->variant_max_subcolumns_count() && - _enable_doc_mode == rhs_type->enable_doc_mode(); + // The max subcolumns count and doc mode only guide how storage shreds a Variant column. + // Execution compares, hashes and serializes Variant values canonically without them, so two + // Variant types with different properties are the same execution type (e.g. hash join keys). + return typeid_cast(&rhs) != nullptr; } int64_t DataTypeVariantV2::get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index e2ecf2cb5e0536..9606b880f7b8c2 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -228,7 +228,8 @@ Status DataTypeArraySerDe::serialize_one_cell_to_hive_text( } Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& data_column = assert_cast(from_column); const auto& offsets = data_column.get_offsets(); @@ -241,7 +242,7 @@ Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, } for (size_t i = start; i < end; ++i) { - RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer)); + RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer, options)); } if (!writer.writeEndArray()) { return Status::InternalError("writeEndArray failed"); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index eaf3644efc6879..31378a61081ede 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -122,7 +122,8 @@ class DataTypeArraySerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index acd409be1af91e..c783245130c2b5 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -1239,8 +1239,8 @@ void DataTypeDecimalSerDe::to_string_batch(const IColumn& column, ColumnStrin template Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb"); } else { @@ -1254,8 +1254,9 @@ Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_co } template -Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb_vector"); } else { @@ -1263,6 +1264,10 @@ Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector(const IColumn& JsonbWriter writer; const auto& data = assert_cast&>(from_column).get_data(); for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); if (!writer.writeDecimal(data[i], precision, scale)) { return Status::InvalidArgument( diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 0b01da4bc32230..fc6263b79c8144 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -92,10 +92,12 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; - Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp index c728532eafb977..6055bb3f19572d 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp @@ -342,7 +342,8 @@ void convert_jsonb_to_rapidjson(const JsonbValue& val, rapidjson::Value& target, } Status DataTypeJsonbSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& jsonb_binary = assert_cast(from_column).get_data_at(row_num); const JsonbDocument* doc = nullptr; RETURN_IF_ERROR( diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.h b/be/src/core/data_type_serde/data_type_jsonb_serde.h index 3a243a1c1a8187..4a50aefa5d09e1 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.h +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.h @@ -75,7 +75,8 @@ class DataTypeJsonbSerDe : public DataTypeStringSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index 1feec24aa5841b..b25011de53e022 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -769,7 +769,8 @@ Status DataTypeMapSerDe::from_string_strict_mode(StringRef& str, IColumn& column } Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& map_column = assert_cast(from_column); const ColumnArray::Offsets64& offsets = map_column.get_offsets(); @@ -803,7 +804,7 @@ Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, i return Status::InternalError("writeKey failed : {}", key_str.to_string()); } // write value - RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer)); + RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index 1404963ecf3e10..81ae4ae4724030 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -118,7 +118,8 @@ class DataTypeMapSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {key_serde, value_serde}; diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index de8c896f903848..55f140217abeba 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -125,14 +125,15 @@ Status DataTypeNullableSerDe::serialize_one_cell_to_hive_text( } Status DataTypeNullableSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& col_null = assert_cast(from_column); if (col_null.is_null_at(row_num)) { writer.writeNull(); } else { RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(col_null.get_nested_column(), - row_num, writer)); + row_num, writer, options)); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index 8be1ece1e61a5f..3e1f0188ef2b19 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -125,7 +125,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 029a5875d2b812..8ea6c7bfe6b1f7 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -1169,8 +1169,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) { template Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } @@ -1183,8 +1183,9 @@ Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_col } template -Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } @@ -1193,6 +1194,10 @@ Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector(const IColumn& f const auto& data = assert_cast(from_column).get_data(); const auto scale = get_scale(); for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); if (!write_to_jsonb_from_number(data[i], writer, scale)) { return Status::InvalidArgument( diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 92828d74a163b8..571888ff34bfba 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -88,10 +88,12 @@ class DataTypeNumberSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; - Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index bd2a09b6b6e68c..b78e5615aa7f3c 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -144,13 +144,18 @@ Status DataTypeSerDe::default_from_string(StringRef& str, IColumn& column) const return deserialize_one_cell_from_json(column, slice, options); } -Status DataTypeSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { const auto size = from_column.size(); JsonbWriter writer; for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); - RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer)); + RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer, options)); to_column.insert_data(writer.getOutput()->getBuffer(), writer.getOutput()->getSize()); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index b4cb8936aa369e..e44bedc1a4911a 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -439,12 +439,15 @@ class DataTypeSerDe { } virtual Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } - virtual Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const; + // Skip rows in source_null_map and insert a default value for each skipped row. + virtual Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const; virtual Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const { diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 2b60cc5f083186..4bc8c51edcea92 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -766,9 +766,9 @@ Status DataTypeStringSerDeBase::write_column_to_orc( } template -Status DataTypeStringSerDeBase::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { +Status DataTypeStringSerDeBase::serialize_column_to_jsonb( + const IColumn& from_column, int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!std::is_same_v) { return Status::NotSupported( "DataTypeStringSerDeBase only supports ColumnString for serialize_column_to_jsonb"); diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index f90b27b1555452..46ca26a6676388 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -178,7 +178,8 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index 04f378104f5d9c..23b6bf3103cb36 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -385,7 +385,8 @@ Status DataTypeStructSerDe::serialize_one_cell_to_hive_text( } Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& struct_column = assert_cast(from_column); if (!writer.writeStartObject()) { @@ -403,7 +404,7 @@ Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column } // write value RETURN_IF_ERROR(elem_serdes_ptrs[i]->serialize_column_to_jsonb(struct_column.get_column(i), - row_num, writer)); + row_num, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index a6a810940bc756..7cb7eac4d57f72 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -114,7 +114,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp index 4e9ae6f872d1cd..20c7ba92936d22 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp @@ -34,6 +34,7 @@ #include "core/arena.h" #include "core/assert_cast.h" #include "core/column/column_const.h" +#include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/column/variant_v2/column_variant_v2_typed_column.h" @@ -48,6 +49,7 @@ #include "core/value/variant/variant_batch_builder.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" +#include "util/jsonb_document.h" #include "util/jsonb_writer.h" #include "util/mysql_row_buffer.h" @@ -479,6 +481,47 @@ void DataTypeVariantV2SerDe::read_one_cell_from_jsonb(IColumn& column, destination(column).insert_encoded_batch(block); } +// to_json and CAST( AS JSON) write Variant values here, with the same session time zone +// as CAST( AS JSON). +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb(const IColumn& from_column, + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { + RETURN_IF_CATCH_EXCEPTION({ + const size_t row = checked_row(row_num); + JsonbWriter document; + visit_variant_v2_values( + from_column, row, row + 1, {}, [](size_t) {}, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, document, {.timezone = options.timezone}); + }); + const JsonbDocument* doc = nullptr; + THROW_IF_ERROR(JsonbDocument::checkAndCreateDocument( + document.getOutput()->getBuffer(), document.getOutput()->getSize(), &doc)); + require_jsonb_write(writer.writeValue(doc->getValue()), "Variant value"); + }); + return Status::OK(); +} + +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { + RETURN_IF_CATCH_EXCEPTION({ + JsonbWriter writer; + visit_variant_v2_values( + from_column, 0, from_column.size(), + source_null_map + ? std::span(source_null_map, from_column.size()) + : std::span(), + [&](size_t) { to_column.insert_default(); }, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, writer, {.timezone = options.timezone}); + to_column.insert_data(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()); + }); + }); + return Status::OK(); +} + namespace { DorisVector json_lengths(const IColumn& column, size_t start, size_t end, diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.h b/be/src/core/data_type_serde/data_type_variant_v2_serde.h index ff2565784b6e6e..c4bc849ca8db6a 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.h +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.h @@ -53,6 +53,13 @@ class DataTypeVariantV2SerDe final : public DataTypeSerDe { const FormatOptions& options) const override; void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; + Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, + JsonbWriter& writer, + const FormatOptions& options) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; diff --git a/be/src/core/value/variant/variant_canonical.cpp b/be/src/core/value/variant/variant_canonical.cpp index 07b8dcc5965fc6..ffa0dcae0c8cc8 100644 --- a/be/src/core/value/variant/variant_canonical.cpp +++ b/be/src/core/value/variant/variant_canonical.cpp @@ -30,9 +30,12 @@ #include #include +#include "common/compare.h" #include "common/exception.h" +#include "core/uint128.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/common/int_exp.h" #include "exec/common/sip_hash.h" #include "util/hash_util.hpp" #include "util/utf8_check.h" @@ -256,77 +259,253 @@ ObjectEntry object_entry_at(VariantRef object, uint32_t index, StringRef previou return {.key = key, .value = child}; } -bool scalar_equals(const NormalizedValue& left, const NormalizedValue& right) { +bool is_numeric(CanonicalKind kind) { + return kind == CanonicalKind::EXACT_INTEGER || kind == CanonicalKind::DECIMAL || + kind == CanonicalKind::FLOATING; +} + +struct ExactNumber { + unsigned __int128 magnitude; + uint8_t scale; + bool negative; +}; + +ExactNumber exact_number(const NormalizedValue& value) { + DCHECK(value.kind == CanonicalKind::EXACT_INTEGER || value.kind == CanonicalKind::DECIMAL); + return {.magnitude = variant_unsigned_magnitude(value.integer), + .scale = static_cast(value.kind == CanonicalKind::DECIMAL ? value.scale : 0), + .negative = value.integer < 0}; +} + +// A Variant decimal scale is validated to [0, 38] before the value is stored +// (variant_scalar.cpp, variant_value.cpp), so the shared exp10 table, which covers 0 to 76, always +// holds these exponents. +UInt256 power_of_five(uint8_t exponent) { + // 10^k = 2^k * 5^k, so 5^k is the table entry shifted right by k, exactly. + return static_cast(common::exp10_i256(exponent)) >> exponent; +} + +UInt256 power_of_ten(uint8_t exponent) { + return static_cast(common::exp10_i256(exponent)); +} + +uint16_t bit_width(const UInt256& value) { + constexpr unsigned LIMB_BITS = sizeof(value.items[0]) * 8; + // big(index) walks the limbs most significant first, so the first non-zero one carries the + // width; scanning limbs costs 4 iterations instead of up to 256 single-bit shifts. + for (unsigned index = 0; index < UInt256::item_count; ++index) { + const auto limb = value.items[UInt256::_impl::big(index)]; + if (limb != 0) { + return static_cast((UInt256::item_count - index) * LIMB_BITS - + static_cast(std::countl_zero(limb))); + } + } + return 0; +} + +int compare_exact_magnitudes(const ExactNumber& left, const ExactNumber& right) { + // Equal scales are the common case - two JSON integers both have scale 0 - and need no + // rescaling, so they never reach the 256-bit path below. + if (left.scale == right.scale) { + return Compare::compare(left.magnitude, right.magnitude); + } + UInt256 scaled_left {left.magnitude}; + UInt256 scaled_right {right.magnitude}; + if (left.scale < right.scale) { + scaled_left *= power_of_ten(right.scale - left.scale); + } else { + scaled_right *= power_of_ten(left.scale - right.scale); + } + return Compare::compare(scaled_left, scaled_right); +} + +struct FloatingNumber { + uint64_t significand; + int16_t binary_exponent; + bool negative; + bool infinity; + bool nan; +}; + +FloatingNumber floating_number(uint64_t bits) { + constexpr uint64_t FRACTION_MASK = (uint64_t {1} << 52) - 1; + constexpr uint16_t EXPONENT_MASK = (uint16_t {1} << 11) - 1; + const auto exponent = static_cast((bits >> 52) & EXPONENT_MASK); + const uint64_t fraction = bits & FRACTION_MASK; + if (exponent == EXPONENT_MASK) { + return {.significand = 0, + .binary_exponent = 0, + .negative = (bits >> 63) != 0, + .infinity = fraction == 0, + .nan = fraction != 0}; + } + if (exponent == 0) { + return {.significand = fraction, + .binary_exponent = -1074, + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; + } + return {.significand = (uint64_t {1} << 52) | fraction, + .binary_exponent = static_cast(static_cast(exponent) - 1023 - 52), + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; +} + +int compare_floating_exact_magnitudes(const FloatingNumber& floating, const ExactNumber& exact) { + DCHECK(!floating.nan && !floating.infinity); + DCHECK_NE(floating.significand, 0); + DCHECK(exact.magnitude != 0); + UInt256 floating_scaled = UInt256(floating.significand) * power_of_five(exact.scale); + UInt256 exact_scaled {exact.magnitude}; + const int16_t binary_shift = floating.binary_exponent + exact.scale; + if (binary_shift >= 0) { + const uint16_t floating_width = bit_width(floating_scaled); + if (floating_width + binary_shift > bit_width(exact_scaled)) { + return 1; + } + floating_scaled <<= binary_shift; + } else { + const uint16_t exact_width = bit_width(exact_scaled); + const auto shift = static_cast(-binary_shift); + if (exact_width + shift > bit_width(floating_scaled)) { + return -1; + } + exact_scaled <<= shift; + } + return Compare::compare(floating_scaled, exact_scaled); +} + +int compare_numeric(const NormalizedValue& left, const NormalizedValue& right) { + if (left.kind != CanonicalKind::FLOATING && right.kind != CanonicalKind::FLOATING) { + const ExactNumber exact_left = exact_number(left); + const ExactNumber exact_right = exact_number(right); + if (exact_left.negative != exact_right.negative) { + return exact_left.negative ? -1 : 1; + } + int result = compare_exact_magnitudes(exact_left, exact_right); + return exact_left.negative ? -result : result; + } + if (left.kind == CanonicalKind::FLOATING && right.kind == CanonicalKind::FLOATING) { + const FloatingNumber floating_left = floating_number(left.floating_bits); + const FloatingNumber floating_right = floating_number(right.floating_bits); + if (floating_left.nan || floating_right.nan) { + return Compare::compare(floating_left.nan, floating_right.nan); + } + const auto value_left = std::bit_cast(left.floating_bits); + const auto value_right = std::bit_cast(right.floating_bits); + return Compare::compare(value_left, value_right); + } + + const bool floating_on_left = left.kind == CanonicalKind::FLOATING; + const FloatingNumber floating = + floating_number((floating_on_left ? left : right).floating_bits); + const ExactNumber exact = exact_number(floating_on_left ? right : left); + int result; + if (floating.nan) { + result = 1; + } else if (floating.infinity) { + result = floating.negative ? -1 : 1; + } else if (exact.magnitude == 0) { + result = floating.negative ? -1 : 1; + } else if (floating.negative != exact.negative) { + result = floating.negative ? -1 : 1; + } else { + result = compare_floating_exact_magnitudes(floating, exact); + if (floating.negative) { + result = -result; + } + } + return floating_on_left ? result : -result; +} + +int scalar_compare(const NormalizedValue& left, const NormalizedValue& right) { + if (is_numeric(left.kind) && is_numeric(right.kind)) { + const int numeric_result = compare_numeric(left, right); + if (numeric_result != 0) { + return numeric_result; + } + return Compare::compare(left.kind, right.kind); + } + if (left.kind != right.kind) { + return Compare::compare(left.kind, right.kind); + } switch (left.kind) { case CanonicalKind::NULL_VALUE: - return true; + return 0; case CanonicalKind::BOOL: - return left.boolean == right.boolean; + return Compare::compare(left.boolean, right.boolean); case CanonicalKind::EXACT_INTEGER: case CanonicalKind::DATE: case CanonicalKind::TIMESTAMP_TZ: case CanonicalKind::TIMESTAMP_NTZ: case CanonicalKind::TIME: - return left.integer == right.integer; + return Compare::compare(left.integer, right.integer); case CanonicalKind::DECIMAL: - return left.integer == right.integer && left.scale == right.scale; + DCHECK(false) << "Decimal should use numeric comparison"; + return 0; case CanonicalKind::FLOATING: - return left.floating_bits == right.floating_bits; + DCHECK(false) << "Floating point should use numeric comparison"; + return 0; case CanonicalKind::STRING: case CanonicalKind::BINARY: - return left.bytes == right.bytes; + return Compare::compare(left.bytes, right.bytes); case CanonicalKind::UUID: - return left.uuid == right.uuid; + return Compare::compare(left.uuid, right.uuid); case CanonicalKind::OBJECT: case CanonicalKind::ARRAY: break; } - DCHECK(false) << "Container reached scalar equality"; - return false; + DCHECK(false) << "Container reached scalar comparison"; + return 0; } -bool equals_node(VariantRef left, VariantRef right, uint32_t depth) { +int compare_node(VariantRef left, VariantRef right, uint32_t depth) { require_depth(depth); require_exact_value(left); require_exact_value(right); const NormalizedValue normalized_left = normalize_value(left); const NormalizedValue normalized_right = normalize_value(right); if (normalized_left.kind != normalized_right.kind) { - return false; + if (!is_numeric(normalized_left.kind) || !is_numeric(normalized_right.kind)) { + return Compare::compare(normalized_left.kind, normalized_right.kind); + } } if (normalized_left.kind == CanonicalKind::ARRAY) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } - for (uint32_t index = 0; index < count; ++index) { - if (!equals_node(left.array_at(index), right.array_at(index), depth + 1)) { - return false; + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { + const int result = compare_node(left.array_at(index), right.array_at(index), depth + 1); + if (result != 0) { + return result; } } - return true; + return Compare::compare(left_count, right_count); } if (normalized_left.kind == CanonicalKind::OBJECT) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); StringRef previous_left; StringRef previous_right; - for (uint32_t index = 0; index < count; ++index) { + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { const ObjectEntry left_entry = object_entry_at(left, index, previous_left, index != 0); const ObjectEntry right_entry = object_entry_at(right, index, previous_right, index != 0); - if (left_entry.key != right_entry.key || - !equals_node(left_entry.value, right_entry.value, depth + 1)) { - return false; + const int key_result = Compare::compare(left_entry.key, right_entry.key); + if (key_result != 0) { + return key_result; + } + const int value_result = compare_node(left_entry.value, right_entry.value, depth + 1); + if (value_result != 0) { + return value_result; } previous_left = left_entry.key; previous_right = right_entry.key; } - return true; + return Compare::compare(left_count, right_count); } - return scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); } template @@ -355,7 +534,7 @@ void update_bytes(Sink& sink, StringRef bytes) { } template -void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { +inline void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { switch (normalized.kind) { case CanonicalKind::NULL_VALUE: return; @@ -1004,7 +1183,11 @@ void VariantCrc32cHashSink::update(const char* data, size_t size) { } bool canonical_equals(VariantRef left, VariantRef right) { - return equals_node(left, right, 0); + return canonical_compare(left, right) == 0; +} + +int canonical_compare(VariantRef left, VariantRef right) { + return compare_node(left, right, 0); } template @@ -1018,10 +1201,21 @@ template void canonical_hash(VariantRef value, VariantCrc3 template void canonical_hash(VariantRef value, VariantCrc32cHashSink& sink); bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right) { + return canonical_compare(left, right) == 0; +} + +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right) { const NormalizedValue normalized_left = VariantScalarAdapter::normalize(left); const NormalizedValue normalized_right = VariantScalarAdapter::normalize(right); - return normalized_left.kind == normalized_right.kind && - scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); +} + +int canonical_compare(const VariantScalarRef& left, VariantRef right) { + require_exact_value(right); + // A scalar never has a container kind, so an object or array on the right is decided by the + // kind order inside scalar_compare(), as compare_node() decides it, without walking the + // container. + return scalar_compare(VariantScalarAdapter::normalize(left), normalize_value(right)); } template diff --git a/be/src/core/value/variant/variant_canonical.h b/be/src/core/value/variant/variant_canonical.h index 286b43f94ec723..15d78e1334613d 100644 --- a/be/src/core/value/variant/variant_canonical.h +++ b/be/src/core/value/variant/variant_canonical.h @@ -35,6 +35,11 @@ struct VariantScalarAdapter; bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right); +// Returns a negative value, zero, or a positive value when left is respectively less than, equal +// to, or greater than right in the canonical Variant total order. Zero is exactly equivalent to +// canonical_equals(). +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right); + template void canonical_hash(const VariantScalarRef& value, Sink& sink); @@ -77,8 +82,28 @@ class VariantCrc32cHashSink { // Compares logical Variant values. Numeric normalization is deliberately limited to the exact // integer domain that canonical Variant decimal16 can encode: [-(10^38-1), +(10^38-1)]. A finite // integral float/double outside that domain remains floating so canonical arena bytes stay valid. +// +// The canonical equality contract for numbers: +// - an integer, a decimal with a zero fraction and an integral float/double in the domain above +// are one value: 1, 1.0 (decimal) and 1.0 (double) are equal; +// - a decimal and a float/double with a fraction are never equal, even when they print the same: +// DECIMAL 1.5 != DOUBLE 1.5. They order by numeric value, and at a numeric tie the decimal +// sorts first. Equating them would need a lossless common form, and the exact value of most +// doubles (0.1 for one) does not fit the 38 digits of a Variant decimal. +// canonical_equals(), canonical_compare() == 0, canonical_hash() and the canonical serialized key +// (GROUP BY, DISTINCT, join build) all implement this one rule, and sort peers follow it. A change +// to the rule has to change all of them together. bool canonical_equals(VariantRef left, VariantRef right); +// Container values are compared lexicographically after scalar normalization. Object keys use +// their canonical byte order. Numeric kinds share a numeric order, with canonical kind as a +// deterministic tie-breaker when two numerically equal values are not canonically equal. +int canonical_compare(VariantRef left, VariantRef right); + +// Compares a typed scalar with an encoded root in the same order, without encoding the scalar: +// canonical_compare(left, right) == canonical_compare(encode(left), right). +int canonical_compare(const VariantScalarRef& left, VariantRef right); + // Hashes canonical logical tokens without re-encoding the value. Supported production sinks are // SipHash, VariantXxHashSink, VariantCrc32HashSink, and VariantCrc32cHashSink; all four are explicit // instantiations of the same traversal skeleton. diff --git a/be/src/exec/sort/sort_block.cpp b/be/src/exec/sort/sort_block.cpp index 69277b0c46aa80..c45b5d95aa53e0 100644 --- a/be/src/exec/sort/sort_block.cpp +++ b/be/src/exec/sort/sort_block.cpp @@ -21,9 +21,17 @@ #include "exec/sort/sort_block.h" #include "core/block/column_with_type_and_name.h" +#include "core/column/variant_v2/column_variant_v2.h" namespace doris { #include "common/compile_check_begin.h" + +void ColumnSorter::sort_column(const ColumnVariantV2& column, EqualFlags& flags, + IColumn::Permutation& perms, EqualRange& range, + bool last_column) const { + _sort_by_default(column, flags, perms, range, last_column); +} + ColumnsWithSortDescriptions get_columns_with_sort_description(const Block& block, const SortDescription& description) { size_t size = description.size(); diff --git a/be/src/exec/sort/sort_block.h b/be/src/exec/sort/sort_block.h index d7da474e246add..19bf3c9a1d0d66 100644 --- a/be/src/exec/sort/sort_block.h +++ b/be/src/exec/sort/sort_block.h @@ -51,6 +51,7 @@ template class ColumnDecimal; template class ColumnVector; +class ColumnVariantV2; } // namespace doris namespace doris { @@ -253,6 +254,8 @@ class ColumnSorter { EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); } + void sort_column(const ColumnVariantV2& column, EqualFlags& flags, IColumn::Permutation& perms, + EqualRange& range, bool last_column) const; void sort_column(const ColumnVarbinary& column, EqualFlags& flags, IColumn::Permutation& perms, EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); @@ -388,7 +391,8 @@ class ColumnSorter { !std::is_same_v && !std::is_same_v && !std::is_same_v && - !std::is_same_v) { + !std::is_same_v && + !std::is_same_v) { auto value_a = column.get_data()[a]; auto value_b = column.get_data()[b]; return Compare::compare(value_a, value_b); diff --git a/be/src/exprs/function/cast/cast_to_jsonb.h b/be/src/exprs/function/cast/cast_to_jsonb.h index ca90ff7d877567..ac834557d45261 100644 --- a/be/src/exprs/function/cast/cast_to_jsonb.h +++ b/be/src/exprs/function/cast/cast_to_jsonb.h @@ -24,6 +24,7 @@ #include "core/value/jsonb_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_string.h" +#include "runtime/runtime_state.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -133,7 +134,7 @@ WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type, } return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t input_rows_count, const NullMap::value_type*) { + uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { CastParameters params; params.is_strict = context->enable_strict_mode(); @@ -231,13 +232,15 @@ WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const Dat } return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t input_rows_count, const NullMap::value_type*) { + uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { // same as to_json function auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; - RETURN_IF_ERROR( - from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column)); + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); + RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, + options, null_map)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); }; diff --git a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp index 7eea84bbdccd6b..f5e6ede566b1b7 100644 --- a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp +++ b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp @@ -425,7 +425,12 @@ Status cast_scalar_to_variant(const ColumnPtr& source, const DataTypePtr& source } ColumnPtr null_map = std::move(nulls); ColumnPtr nullable = ColumnNullable::create(source, null_map); - *output = ColumnVariantV2::create_typed(std::move(nullable), source_type); + auto typed = ColumnVariantV2::create_typed(std::move(nullable), source_type); + // A scalar column can hold values Variant cannot, such as a string that is not UTF-8. Reject + // them here, as encoding would, so the typed state never compares or hashes a value that its + // encoded form could not hold. The CAST wrapper turns the exception into the statement error. + typed->validate_typed_rows(); + *output = std::move(typed); return Status::OK(); } diff --git a/be/src/exprs/function/function_to_json.cpp b/be/src/exprs/function/function_to_json.cpp index bae1f6767af174..d8956c0d3d2d75 100644 --- a/be/src/exprs/function/function_to_json.cpp +++ b/be/src/exprs/function/function_to_json.cpp @@ -16,7 +16,10 @@ // under the License. #include "core/data_type/data_type_jsonb.h" +#include "core/data_type_serde/data_type_serde.h" #include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "runtime/runtime_state.h" namespace doris { #include "common/compile_check_begin.h" @@ -43,8 +46,10 @@ class FunctionToJson : public IFunction { auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; - RETURN_IF_ERROR( - from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column)); + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); + RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, + options)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); } diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index efb68f5f6325c2..e054aba7dd812b 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -872,12 +872,12 @@ void append_value(VariantRef value, const DataTypePtr& target_type, IColumn* tar Status stringify_complex_column(const DataTypePtr& source_type, const ColumnNullable& source, ColumnPtr* result) { + DataTypeSerDe::FormatOptions options; auto jsonb = ColumnString::create(); RETURN_IF_ERROR(source_type->get_serde()->serialize_column_to_jsonb_vector( - source.get_nested_column(), *jsonb)); + source.get_nested_column(), *jsonb, options)); auto strings = ColumnString::create(); - DataTypeSerDe::FormatOptions options; DataTypeJsonb().get_serde()->to_string_batch(*jsonb, *strings, options); *result = ColumnNullable::create(std::move(strings), source.get_null_map_column().clone_resized(source.size())); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index fc214943541985..6c9badeaa375b6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -285,7 +285,8 @@ public abstract class Type { variantSubTypes.add(DECIMAL256); variantSubTypes.add(DATEV2); variantSubTypes.add(DATETIMEV2); - variantSubTypes.add(TIMESTAMP_TZ); + // TIMESTAMP_TZ is deliberately absent. This list is only checked when a column or + // function type is defined, so tables that already declare a TIMESTAMPTZ path keep working. variantSubTypes.add(IPV4); variantSubTypes.add(IPV6); variantSubTypes.add(STRING); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index 032894cf8602a6..effbd78ee19432 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -142,7 +142,9 @@ public static PushDownContext createPushDownContextForHashJoin(Expression srcExp } public boolean isValid() { - return finalTarget != null && probeSlot != null; + return finalTarget != null && probeSlot != null + && !srcExpr.getDataType().isVariantType() + && !probeExpr.getDataType().isVariantType(); } public PushDownContext withNewProbeExpression(Expression newProbe) { @@ -185,6 +187,9 @@ public Boolean visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapS @Override public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) { + if (ctx.srcExpr.getDataType().isVariantType() || ctx.probeExpr.getDataType().isVariantType()) { + return false; + } if (scan instanceof PhysicalSchemaScan) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 50826ece82ae81..7d23671126f6f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -46,7 +46,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; -import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VariantType; import com.google.common.collect.Sets; @@ -149,7 +148,7 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { LogicalAggregate agg = (LogicalAggregate) plan; for (Expression groupBy : agg.getGroupByExpressions()) { if (groupBy.getDataType().isObjectType() - || isLegacyVariant(groupBy.getDataType()) + || VariantType.isLegacyVariant(groupBy.getDataType()) || groupBy.getDataType().isVarBinaryType()) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } @@ -157,14 +156,16 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } else if (plan instanceof LogicalSort) { LogicalSort sort = (LogicalSort) plan; for (OrderKey orderKey : sort.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getExpr().getDataType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } } else if (plan instanceof LogicalTopN) { LogicalTopN topN = (LogicalTopN) plan; for (OrderKey orderKey : topN.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getExpr().getDataType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } @@ -174,24 +175,26 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { return; } WindowExpression windowExpression = (WindowExpression) ((Alias) a).child(); - if (windowExpression.getOrderKeys().stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + if (windowExpression.getOrderKeys().stream().anyMatch(orderKey -> + orderKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } - if (windowExpression.getPartitionKeys().stream().anyMatch(( - partitionKey -> partitionKey.getDataType().isObjectOrVariantType()))) { + if (windowExpression.getPartitionKeys().stream().anyMatch(partitionKey -> + partitionKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(partitionKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } }); } else if (plan instanceof LogicalSetOperation && ((LogicalSetOperation) plan).getQualifier() == Qualifier.DISTINCT) { - if (plan.getOutput().stream().anyMatch(output -> isLegacyVariant(output.getDataType()))) { + if (plan.getOutput().stream().anyMatch(output -> VariantType.isLegacyVariant(output.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } else if (plan instanceof LogicalJoin) { LogicalJoin join = (LogicalJoin) plan; for (Expression conjunct : join.getHashJoinConjuncts()) { - if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVariantType())) { + if (conjunct.anyMatch(e -> VariantType.isLegacyVariant(((Expression) e).getDataType()))) { throw new AnalysisException("variant type could not in join equal conditions: " + conjunct.toSql()); } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( @@ -199,7 +202,7 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } } for (Expression conjunct : join.getMarkJoinConjuncts()) { - if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVariantType())) { + if (conjunct.anyMatch(e -> VariantType.isLegacyVariant(((Expression) e).getDataType()))) { throw new AnalysisException("variant type could not in join equal conditions: " + conjunct.toSql()); } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( @@ -209,10 +212,6 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } } - private boolean isLegacyVariant(DataType dataType) { - return dataType instanceof VariantType && !((VariantType) dataType).isExecutionV2(); - } - private void checkMatchIsUsedCorrectly(Plan plan) { for (Expression expression : plan.getExpressions()) { if (expression instanceof Match) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java index 762896a240844b..9c1853d7a9911d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.types.VariantType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -81,8 +82,9 @@ private static Plan eliminateWindow(LogicalWindow window) { Alias alias = (Alias) expr; WindowExpression windowExpression = (WindowExpression) alias.child(); List orderExpressions = windowExpression.getOrderKeys(); - if (orderExpressions.stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + if (orderExpressions.stream().anyMatch(orderKey -> + orderKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } List orderKeys = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java index ce262e8610cb4e..18e99c3c7e7888 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java @@ -19,9 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.JsonType; import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.ComplexDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -44,11 +42,6 @@ static boolean isExplicitlyCastable(DataType signatureType, DataType realType) { /** isExplicitlyCastable */ static boolean isPrimitiveExplicitlyCastable(DataType signatureType, DataType realType) { - if (signatureType instanceof JsonType - && realType instanceof VariantType - && ((VariantType) realType).isExecutionV2()) { - return false; - } if (signatureType instanceof AnyDataType || signatureType instanceof FollowToAnyDataType || signatureType.isAssignableFrom(realType)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java index 6345f29f036c81..e53465515d898b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java @@ -19,7 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.NullOrIdenticalSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -35,7 +35,7 @@ * to_json convert type to json */ public class JsonHash extends ScalarFunction - implements UnaryExpression, NullOrIdenticalSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(JsonType.INSTANCE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java index a6e571c23231c6..db9830cea694c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VariantType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -96,7 +97,8 @@ public ToJson withChildren(List children) { @Override public List getSignatures() { DataType firstChildType = child(0).getDataType(); - if (firstChildType.isStructType() || firstChildType.isArrayType()) { + if (firstChildType.isStructType() || firstChildType.isArrayType() + || (firstChildType instanceof VariantType && ((VariantType) firstChildType).isExecutionV2())) { return ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE).args(firstChildType)); } if (firstChildType.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 0acc6340e916ef..d9ec97fb155064 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IntegralDivide; @@ -224,11 +225,8 @@ public static Optional implicitCast(DataType input, DataType expected) } return Optional.of(new StructType(newFields)); } else if (input instanceof VariantType - && ((VariantType) input).isExecutionV2() && expected instanceof JsonType) { - // JSON functions require users to make this representation change explicit. - return Optional.empty(); - } else if (input instanceof VariantType && (expected.isNumericType() || expected.isStringLikeType())) { - // variant could implicit cast to numric types and string like types + && (expected.isNumericType() || expected.isStringLikeType() || expected.isJsonType())) { + // variant could implicit cast to numric types, string like types and json return Optional.of(expected); } else { return implicitCastPrimitive(input, expected); @@ -1400,16 +1398,29 @@ public static Expression processComparisonPredicate(ComparisonPredicate comparis Expression left = comparisonPredicate.left(); Expression right = comparisonPredicate.right(); - boolean leftIsVariantV2 = left.getDataType() instanceof VariantType + boolean leftIsVariant = left.getDataType() instanceof VariantType && ((VariantType) left.getDataType()).isExecutionV2(); - boolean rightIsVariantV2 = right.getDataType() instanceof VariantType + boolean rightIsVariant = right.getDataType() instanceof VariantType && ((VariantType) right.getDataType()).isExecutionV2(); - boolean isDirectVariantSubpathScalarComparison = leftIsVariantV2 != rightIsVariantV2 - && ((leftIsVariantV2 && left instanceof ElementAt) - || (rightIsVariantV2 && right instanceof ElementAt)); - if ((leftIsVariantV2 || rightIsVariantV2) + // V2 equality is shared by scalar predicates and canonical hash join keys. Keep + // ordering and mixed Variant/scalar comparisons on their existing coercion paths. + if (leftIsVariant && rightIsVariant && comparisonPredicate instanceof EqualPredicate) { + return comparisonPredicate; + } + // A bare NULL has no type of its own, so it takes the Variant type of the other side and + // `v = NULL`, `v != NULL` and `v <=> NULL` become the Variant equality above. + if (comparisonPredicate instanceof EqualPredicate && leftIsVariant != rightIsVariant + && (leftIsVariant ? right : left).getDataType().isNullType()) { + DataType variantDataType = leftIsVariant ? left.getDataType() : right.getDataType(); + return comparisonPredicate.withChildren(castIfNotSameType(left, variantDataType), + castIfNotSameType(right, variantDataType)); + } + boolean isDirectVariantSubpathScalarComparison = leftIsVariant != rightIsVariant + && ((leftIsVariant && left instanceof ElementAt) + || (rightIsVariant && right instanceof ElementAt)); + if ((leftIsVariant || rightIsVariant) && !isDirectVariantSubpathScalarComparison) { - DataType variantDataType = leftIsVariantV2 + DataType variantDataType = leftIsVariant ? left.getDataType() : right.getDataType(); throw new AnalysisException("data type " + variantDataType + " could not used in ComparisonPredicate " + comparisonPredicate.toSql() From 4501892e7a01092a77a3da883f636bb3f5f2f755 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sat, 10 Oct 2026 18:20:37 +0800 Subject: [PATCH 2/5] [test](variant) Cover branch-4.2 Variant V2 operations and V1 compatibility ### What problem does this PR solve? Related PR: #67675 Problem Summary: Backport relational, JSON, canonical ordering and runtime-filter coverage. Enable Variant V2 temporarily in its suites and restore the prior configuration. Retain legacy Variant canonical and scalar coercion coverage and add a regression for V1 ordering, join restrictions, comparison and JSON extraction. Adapt benchmark phases to temporarily enable V2 while ordinary load runs retain the configured Variant version. ### Release note None ### Check List (For Author) - Test: 114 related ASAN BE unit tests; 86 FE unit tests; 25 regression suites passed. - Expanded BE run: 858 passed, 5 skipped, 1 unchanged V1 byte_size golden mismatch; reproduced alone. - Golden for the new V1 compatibility suite generated by run-regression-test.sh. - Behavior changed: No, test coverage and benchmark configuration only. - Does this need documentation: No, benchmark notes updated in this commit. --- .../core/column/column_variant_v2_test.cpp | 547 +++++++++++++++- .../data_type_variant_v2_serde_jsonb_test.cpp | 156 +++++ .../data_type_write_to_jsonb_test.cpp | 36 +- .../function/cast/cast_variant_v2_to_test.cpp | 37 ++ be/test/exprs/variant_v2_execution_test.cpp | 79 +++ .../util/variant/variant_canonical_test.cpp | 119 ++++ .../org/apache/doris/catalog/TypeTest.java | 8 + .../load/VariantLoadParseInjectionTest.java | 13 + .../postprocess/RuntimeFilterTest.java | 94 +++ ...ExpressionAnalyzerVariantAutoCastTest.java | 40 -- .../analysis/VariantEqualityContextTest.java | 107 ++- .../nereids/util/TypeCoercionUtilsTest.java | 92 +-- .../data/nereids_p0/join/test_join_on.out | 4 + .../variant_p0/test_variant_bool_path.out | 54 ++ .../data/variant_p0/test_variant_join.out | 160 +++++ .../test_variant_join_type_properties.out | 53 ++ .../test_variant_json_functions.out | 53 ++ ...test_variant_ordering_comparison_error.out | 102 +++ .../test_variant_relational_corners.out | 342 ++++++++++ .../test_variant_subfield_comparison.out | 610 ++++++++++++++++++ .../test_variant_timestamptz_sparse.out | 7 - ...st_variant_v1_relational_compatibility.out | 7 + .../test_variant_window_partition.out | 55 ++ .../v2/test_variant_equality_contexts.out | 18 + .../nereids_p0/join/test_join_on.groovy | 10 +- .../predefined_typed_to_sparse_1shard.groovy | 1 - .../predefine/test_predefine_ddl.groovy | 30 + .../variant_p0/test_variant_bool_path.groovy | 116 ++++ .../variant_p0/test_variant_join.groovy | 92 +++ .../test_variant_join_type_properties.groovy | 90 +++ .../test_variant_json_functions.groovy | 102 +++ ...t_variant_ordering_comparison_error.groovy | 100 ++- .../test_variant_relational_corners.groovy | 124 ++++ .../test_variant_subfield_comparison.groovy | 182 ++++++ .../test_variant_timestamptz_sparse.groovy | 103 --- ...variant_v1_relational_compatibility.groovy | 60 ++ .../test_variant_window_partition.groovy | 106 +++ .../v2/test_variant_equality_contexts.groovy | 26 +- .../suites/variant_p2/RELATIONAL_BENCHMARK.md | 101 +++ regression-test/suites/variant_p2/load.groovy | 290 +++++---- .../variant_p2/relational_performance.groovy | 172 +++++ .../variant_p2/run_relational_benchmark.py | 211 ++++++ 42 files changed, 4279 insertions(+), 430 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java create mode 100644 regression-test/data/variant_p0/test_variant_bool_path.out create mode 100644 regression-test/data/variant_p0/test_variant_join.out create mode 100644 regression-test/data/variant_p0/test_variant_join_type_properties.out create mode 100644 regression-test/data/variant_p0/test_variant_json_functions.out create mode 100644 regression-test/data/variant_p0/test_variant_relational_corners.out create mode 100644 regression-test/data/variant_p0/test_variant_subfield_comparison.out delete mode 100644 regression-test/data/variant_p0/test_variant_timestamptz_sparse.out create mode 100644 regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out create mode 100644 regression-test/data/variant_p0/test_variant_window_partition.out create mode 100644 regression-test/suites/variant_p0/test_variant_bool_path.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_join.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_join_type_properties.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_json_functions.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_relational_corners.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy delete mode 100644 regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy create mode 100644 regression-test/suites/variant_p0/test_variant_window_partition.groovy create mode 100644 regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md create mode 100644 regression-test/suites/variant_p2/relational_performance.groovy create mode 100644 regression-test/suites/variant_p2/run_relational_benchmark.py diff --git a/be/test/core/column/column_variant_v2_test.cpp b/be/test/core/column/column_variant_v2_test.cpp index 9f5ff6ce1d27de..2f21e957c749a4 100644 --- a/be/test/core/column/column_variant_v2_test.cpp +++ b/be/test/core/column/column_variant_v2_test.cpp @@ -63,12 +63,14 @@ #include "exec/common/hash_table/hash_map_context.h" #include "exec/common/hash_table/string_hash_map.h" #include "exec/common/sip_hash.h" +#include "exec/sort/sort_block.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" #include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" #include "util/jsonb_writer.h" #include "util/variant/variant_test_utils.h" +#include "vec/core/hybrid_sorter.h" namespace doris { namespace { @@ -284,6 +286,50 @@ class CountingShreddedState final : public VariantShreddedState { ColumnVariantV2::MutablePtr _serialized; }; +// A complete shredded state over known rows that counts how often its canonical rows are asked for. +class MaterializingShreddedState final : public VariantShreddedState { +public: + MaterializingShreddedState(ColumnPtr rows, std::shared_ptr materialized_calls) + : _rows(std::move(rows)), _materialized_calls(std::move(materialized_calls)) {} + + size_t size() const override { return _rows->size(); } + size_t byte_size() const override { return 0; } + size_t allocated_bytes() const override { return 0; } + void sanity_check() const override {} + void for_each_subcolumn(const IColumn::ImutableColumnCallback&) const override {} + std::shared_ptr filter(const IColumn::Filter&, ssize_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_range(size_t, size_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_indices(const uint32_t*, + const uint32_t*) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + bool can_materialize() const override { return true; } + bool try_append(const VariantShreddedState&) override { return false; } + std::optional find_typed_value( + std::span) const override { + return std::nullopt; + } + std::optional find_normalized_value( + std::span) const override { + return std::nullopt; + } + const ColumnVariantV2& materialized_column() const override { + ++*_materialized_calls; + return assert_cast(*_rows); + } + const ColumnVariantV2& serialized_column() const override { + return assert_cast(*_rows); + } + +private: + ColumnPtr _rows; + std::shared_ptr _materialized_calls; +}; + class ValidatingShreddedState final : public VariantShreddedState { public: enum class TypedLookup { MISS, ERROR }; @@ -587,6 +633,20 @@ void validate_encoded_column(const ColumnVariantV2& column) { } } +// The encoded twin of a typed column. insert_range_from() into an empty column keeps the source's +// typed state, so a test that wants the other representation has to encode explicitly; both +// physical states are asserted so a variable name cannot stand in for coverage. +ColumnVariantV2::MutablePtr encoded_copy_of(const ColumnVariantV2& typed) { + EXPECT_TRUE(typed.is_typed()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(typed, 0, typed.size()); + encoded->ensure_encoded(); + EXPECT_FALSE(encoded->is_typed()); + EXPECT_FALSE(encoded->is_shredded()); + EXPECT_EQ(encoded->size(), typed.size()); + return encoded; +} + void ensure_typed_fields_match_direct_encoding(ColumnVariantV2& column) { ASSERT_TRUE(column.is_typed()); std::vector fields(column.size()); @@ -1751,6 +1811,47 @@ TEST(ColumnVariantV2Test, CompositeShreddedSizeDoesNotRecountSegments) { EXPECT_EQ(*size_calls, 0); } +TEST(ColumnVariantV2Test, EraseRemovesRowsInEveryState) { + // Encoded rows with two metadata dictionaries: the ids of the remaining rows stay valid. + auto encoded = ColumnVariantV2::create(); + for (const char* json : {R"({"a":1})", "2", R"({"b":3})", "4", R"("five")"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + encoded->erase(0, 2); + ASSERT_EQ(encoded->size(), 3); + EXPECT_EQ(json_at(*encoded, 0), R"({"b":3})"); + EXPECT_EQ(json_at(*encoded, 2), R"("five")"); + encoded->erase(1, 1); + ASSERT_EQ(encoded->size(), 2); + EXPECT_EQ(json_at(*encoded, 1), R"("five")"); + encoded->erase(2, 0); + EXPECT_EQ(encoded->size(), 2); + validate_encoded_column(*encoded); + + constexpr std::array VALUES {7, 8, 9, 10}; + constexpr std::array NULLS {0, 1, 0, 0}; + auto typed = typed_int32(VALUES, NULLS); + typed->erase(0, 2); + EXPECT_TRUE(typed->is_typed()); + constexpr std::array REMAINING {9, 10}; + constexpr std::array REMAINING_NULLS {0, 0}; + expect_int32_rows(*typed, REMAINING, REMAINING_NULLS); + + auto rows = ColumnVariantV2::create(); + for (const char* json : {"1", "2", "3"}) { + insert_encoded_field(*rows, encode_json(json)); + } + auto calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(rows->get_ptr(), calls)); + shredded->erase(0, 1); + EXPECT_FALSE(shredded->is_shredded()); + ASSERT_EQ(shredded->size(), 2); + EXPECT_EQ(json_at(*shredded, 0), "2"); + EXPECT_EQ(json_at(*shredded, 1), "3"); + EXPECT_EQ(rows->size(), 3); +} + TEST(ColumnVariantV2Test, CompositeValidatesSegmentsAfterFirstTypedMiss) { auto first_calls = std::make_shared(0); auto second_calls = std::make_shared(0); @@ -2676,6 +2777,30 @@ TEST(ColumnVariantV2Test, TypedEnsureEncodedAllScalarMappings) { ASSERT_EQ(zero_dates.size(), 2); EXPECT_EQ(zero_dates.get_data()[0], valid_date); EXPECT_EQ(zero_dates.get_data()[1], invalid_date); + + // validate_typed_rows() is the check CAST runs on arbitrary scalars. It reads the rows the + // typed null map leaves visible and nothing else. + EXPECT_THROW(zero_date->validate_typed_rows(), Exception); + auto masked_nested = ColumnDateV2::create(); + masked_nested->insert_value(valid_date); + masked_nested->insert_value(invalid_date); + auto masked_nulls = ColumnUInt8::create(); + masked_nulls->insert_value(0); + masked_nulls->insert_value(1); + auto masked = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(masked_nested), std::move(masked_nulls)), + std::make_shared()); + EXPECT_NO_THROW(masked->validate_typed_rows()); + masked->ensure_encoded(); + validate_encoded_column(*masked); + EXPECT_TRUE(masked->get_value_ref(1).is_null()); + + constexpr std::array INVALID_UTF8_ROWS {"ok", "\xFF"}; + constexpr std::array INVALID_UTF8_VISIBLE {0, 0}; + constexpr std::array INVALID_UTF8_HIDDEN {0, 1}; + EXPECT_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_VISIBLE)->validate_typed_rows(), + Exception); + EXPECT_NO_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_HIDDEN)->validate_typed_rows()); } TEST(ColumnVariantV2Test, TypedRowTransformsCloneAndCow) { @@ -2813,6 +2938,8 @@ TEST(ColumnVariantV2Test, TypedMutatorsDetachSharedNestedColumn) { constexpr std::array FIRST_TWO_NULLS {}; exercise([](ColumnVariantV2& column) { column.pop_back(1); }, FIRST_TWO, FIRST_TWO_NULLS); exercise([](ColumnVariantV2& column) { column.resize(2); }, FIRST_TWO, FIRST_TWO_NULLS); + constexpr std::array LAST_TWO {2, 3}; + exercise([](ColumnVariantV2& column) { column.erase(0, 1); }, LAST_TWO, FIRST_TWO_NULLS); constexpr std::array FILTERED {1, 3}; constexpr std::array FILTERED_NULLS {}; exercise( @@ -3035,6 +3162,45 @@ TEST(ColumnVariantV2Test, MixedEncodedTypedInsertAndGatherPreserveCanonicalRows) expect_canonical_rows_equal(*typed, *encoded); } +TEST(ColumnVariantV2Test, NullableBatchSerializationMatchesRowSerialization) { + constexpr std::array VALUES {42, 0, -7}; + constexpr std::array NULLS {0, 1, 0}; + auto typed = typed_int32(VALUES, NULLS); + auto encoded = ColumnVariantV2::create(); + insert_encoded_field(*encoded, encoded_integer(42, 4)); + insert_encoded_field(*encoded, encode_json("null")); + insert_encoded_field(*encoded, encoded_integer(-7, 4)); + for (const auto* column : {typed.get(), encoded.get()}) { + for (const bool has_null : {false, true}) { + std::array buffers; + std::array keys; + for (size_t row = 0; row < 3; ++row) { + buffers[row].assign(8 + column->get_max_row_byte_size(), '\0'); + buffers[row][0] = 'x'; + keys[row] = {buffers[row].data(), 1}; + } + constexpr std::array NO_NULLS {0, 0, 0}; + auto nullable = + ColumnNullable::create(column->clone_resized(3), ColumnUInt8::create(3, 0)); + for (size_t row = 0; row < 3; ++row) { + nullable->get_null_map_data()[row] = has_null ? NULLS[row] : NO_NULLS[row]; + } + nullable->serialize_vec(keys.data(), 3); + for (size_t row = 0; row < 3; ++row) { + std::string expected = "x"; + const bool is_null = has_null && NULLS[row]; + expected.push_back(is_null ? 1 : 0); + if (!is_null) { + std::string cell(column->serialize_size_at(row), '\0'); + column->serialize_impl(cell.data(), row); + expected += cell; + } + EXPECT_EQ(std::string(keys[row].data, keys[row].size), expected); + } + } + } +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive E/T adapter matrix. TEST(ColumnVariantV2Test, TypedCanonicalHashCrcAndArenaMatchEncoded) { constexpr std::array VALUES {42, 0, -7}; @@ -3393,21 +3559,309 @@ TEST(ColumnVariantV2Test, ETCrossCheckTemporalClassMatrix) { ntz_representations[2], 0); } -TEST(ColumnVariantV2Test, TypedPhysicalAndOrderingInterfacesStayUnsupported) { - constexpr std::array VALUES {1}; - constexpr std::array NULLS {0}; +TEST(ColumnVariantV2Test, TypedComparisonBoundariesMatchEncodedWithoutMaterializing) { + auto check = [](const ColumnVariantV2::MutablePtr& typed) { + ASSERT_TRUE(typed->is_typed()); + const auto* original = &typed->typed_column(); + auto encoded = encoded_copy_of(*typed); + ASSERT_FALSE(encoded->is_typed()); + for (size_t right = 0; right < typed->size(); ++right) { + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + const int expected = encoded->compare_at(left, right, *encoded, hint); + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), expected); + EXPECT_EQ(typed->compare_at(left, right, *encoded, hint), expected); + EXPECT_EQ(encoded->compare_at(left, right, *typed, hint), expected); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(right, *typed, hint, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(right, *encoded, hint, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + } + // Prove the tested input did not silently become encoded during comparison. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + check(ColumnVariantV2::create_typed( + nullable_fixed({0, -1, 9007199254740993LL, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-std::numeric_limits::infinity(), -0.0, 0.0, 1.5, + std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + std::bit_cast(uint64_t {0xfff8000000000001ULL}), 0.0}, + {0, 0, 0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 2, {Decimal64 {-150}, Decimal64 {0}, Decimal64 {150}, Decimal64 {0}}, + {0, 0, 0, 1}), + std::make_shared(18, 2))); + const auto date = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 2, 29)); + const auto next = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 3, 1)); + check(ColumnVariantV2::create_typed(nullable_fixed>( + {date, next, date}, {0, 0, 1}), + std::make_shared())); + // Address order differs from textual Variant order for these values. + IPv4 ip4_a {}, ip4_b {}; + ASSERT_TRUE(IPv4Value::from_string(ip4_a, "10.0.0.2")); + ASSERT_TRUE(IPv4Value::from_string(ip4_b, "10.0.0.10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip4_a, ip4_b, ip4_a}, {0, 0, 1}), + std::make_shared())); + IPv6 ip6_a {}, ip6_b {}; + ASSERT_TRUE(IPv6Value::from_string(ip6_a, "2001:db8::2")); + ASSERT_TRUE(IPv6Value::from_string(ip6_b, "2001:db8::10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip6_a, ip6_b, ip6_a}, {0, 0, 1}), + std::make_shared())); + // LARGEINT magnitudes above 10^38 - 1 are represented as Variant strings. + const __int128 limit = power_of_ten_i128(38); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-limit, Int128 {0}, limit - 1, limit, -(limit - 1), Int128 {0}}, + {0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_fixed({1, 0, 1}, {0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed({-1, 0, 127, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-1.25F, 0.5F, std::numeric_limits::quiet_NaN(), 0.0F}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 4, + {Decimal128V3 {Int128 {-15000}}, Decimal128V3 {Int128 {0}}, + Decimal128V3 {Int128 {15000}}, Decimal128V3 {Int128 {0}}}, + {0, 0, 0, 1}), + std::make_shared(38, 4))); + const auto datetime_a = + DateV2Value::create_from_olap_datetime(19700101000001ULL); + const auto datetime_b = + DateV2Value::create_from_olap_datetime(20240229123456ULL); + check(ColumnVariantV2::create_typed( + nullable_fixed>( + {datetime_b, datetime_a, datetime_b}, {0, 0, 1}), + std::make_shared(6))); + // Bytes at or above 0x80 pin the unsigned string order. + constexpr std::array STRINGS {"a", "\xC3\xA9", "z", ""}; + constexpr std::array STRING_NULLS {0, 0, 0, 1}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, TypedComparisonAcrossTypeParametersMatchesEncoded) { + // DECIMAL(18,2) and DECIMAL(18,3) share a column type but not a scale. + auto left = ColumnVariantV2::create_typed(nullable_decimal( + 2, {Decimal64 {150}, Decimal64 {-5}}, {0, 0}), + std::make_shared(18, 2)); + auto right = + ColumnVariantV2::create_typed(nullable_decimal( + 3, {Decimal64 {1400}, Decimal64 {-50}}, {0, 0}), + std::make_shared(18, 3)); + auto left_encoded = encoded_copy_of(*left); + auto right_encoded = encoded_copy_of(*right); + for (size_t right_row = 0; right_row < right->size(); ++right_row) { + for (size_t left_row = 0; left_row < left->size(); ++left_row) { + EXPECT_EQ(left->compare_at(left_row, right_row, *right, 1), + left_encoded->compare_at(left_row, right_row, *right_encoded, 1)); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(left->size(), 0); + std::vector encoded_cmp(left->size(), 0); + IColumn::Filter typed_filter(left->size(), 0); + IColumn::Filter encoded_filter(left->size(), 0); + left->compare_internal(right_row, *right, 1, direction, typed_cmp, typed_filter.data()); + left_encoded->compare_internal(right_row, *right_encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } +} + +TEST(ColumnVariantV2Test, TypedAgainstEncodedRootsMatchesEncoded) { + // The encoded side holds every kind of root a typed scalar can meet, including containers and + // canonical peers of the typed values (2 and 2.0, 1.5 as a decimal). + auto encoded_roots = ColumnVariantV2::create(); + for (const char* json : + {"null", "true", "1", "2", "2.0", "1.5", R"("a")", R"("2")", "[2]", R"({"k":2})"}) { + insert_encoded_field(*encoded_roots, encode_json(json)); + } + ASSERT_FALSE(encoded_roots->is_typed()); + + auto check = [&](const ColumnVariantV2::MutablePtr& typed) { + auto encoded = encoded_copy_of(*typed); + const auto* original = &typed->typed_column(); + for (size_t row = 0; row < typed->size(); ++row) { + for (size_t root = 0; root < encoded_roots->size(); ++root) { + const int expected = encoded->compare_at(row, root, *encoded_roots, 1); + EXPECT_EQ(typed->compare_at(row, root, *encoded_roots, 1), expected) + << "row=" << row << " root=" << root; + EXPECT_EQ(encoded_roots->compare_at(root, row, *typed, 1), -expected) + << "row=" << row << " root=" << root; + } + } + for (size_t root = 0; root < encoded_roots->size(); ++root) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(root, *encoded_roots, 1, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(root, *encoded_roots, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + // Comparing must not change the state of the typed column. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + constexpr std::array INTEGERS {1, 2, 3, 0}; + constexpr std::array INTEGER_NULLS {0, 0, 0, 1}; + check(typed_int32(INTEGERS, INTEGER_NULLS)); + check(ColumnVariantV2::create_typed( + nullable_fixed({1.5, 2.0, std::numeric_limits::quiet_NaN()}, + {0, 0, 0}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_decimal( + 1, {Decimal64 {15}, Decimal64 {20}}, {0, 0}), + std::make_shared(18, 1))); + constexpr std::array STRINGS {"a", "2", "b"}; + constexpr std::array STRING_NULLS {0, 0, 0}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, ShreddedComparisonMatchesEncodedAndStaysShredded) { + auto encoded = ColumnVariantV2::create(); + for (const char* json : {"2", "1.5", R"("a")", R"({"k":1})", "null"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + auto materialized_calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(encoded->get_ptr(), materialized_calls)); + ASSERT_TRUE(shredded->is_shredded()); + constexpr std::array INTEGERS {1, 2, 3}; + constexpr std::array NOT_NULL {0, 0, 0}; + auto typed = typed_int32(INTEGERS, NOT_NULL); + auto typed_encoded = encoded_copy_of(*typed); + + for (size_t left = 0; left < encoded->size(); ++left) { + for (size_t right = 0; right < encoded->size(); ++right) { + const int expected = encoded->compare_at(left, right, *encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *shredded, 1), expected); + EXPECT_EQ(shredded->compare_at(left, right, *encoded, 1), expected); + EXPECT_EQ(encoded->compare_at(left, right, *shredded, 1), expected); + } + for (size_t right = 0; right < typed->size(); ++right) { + const int expected = encoded->compare_at(left, right, *typed_encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *typed, 1), expected); + EXPECT_EQ(typed->compare_at(right, left, *shredded, 1), -expected); + } + } + // Comparison reads the rows the state materializes (a real state caches them); the column + // itself stays shredded, and the typed peer stays typed. + EXPECT_GT(*materialized_calls, 0); + EXPECT_TRUE(shredded->is_shredded()); + EXPECT_TRUE(typed->is_typed()); +} + +TEST(ColumnVariantV2Test, TypedNaNOrderingIgnoresNullDirectionHint) { + auto typed = ColumnVariantV2::create_typed( + nullable_fixed({1.0, std::numeric_limits::quiet_NaN()}, {0, 0}), + std::make_shared()); + auto encoded = encoded_copy_of(*typed); + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), + encoded->compare_at(left, right, *encoded, hint)); + } + } + } +} + +TEST(ColumnVariantV2Test, TypedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { + constexpr std::array VALUES {2, 99, 1, 3}; + constexpr std::array NULLS {0, 1, 0, 0}; auto typed = typed_int32(VALUES, NULLS); expect_not_implemented([&] { static_cast(typed->get_data_at(0)); }, "intentionally unsupported"); + auto encoded = encoded_copy_of(*typed); + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + for (int nan_direction_hint : {-1, 1}) { + EXPECT_EQ(typed->compare_at(left, right, *typed, nan_direction_hint), + encoded->compare_at(left, right, *encoded, nan_direction_hint)); + } + } + } + for (size_t right = 0; right < typed->size(); ++right) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(encoded->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(encoded->size(), 0); + typed->compare_internal(right, *typed, 1, direction, typed_cmp, typed_filter.data()); + encoded->compare_internal(right, *encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { typed->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); + typed->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {1, 2, 0, 3})); + typed->get_permutation(true, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {3, 0})); expect_not_implemented([&] { typed->replace_column_data(*typed, 0); }, "intentionally unsupported"); EXPECT_TRUE(typed->is_typed()); } +TEST(ColumnVariantV2Test, PermutationLimitSwitchesBetweenPartialAndFullSort) { + // get_permutation only uses std::partial_sort while the limit stays small relative to the row + // count, the threshold ColumnVector applies as well; a larger limit falls back to the full + // sort. Both branches must order the leading rows the same way. + constexpr std::array VALUES {9, 3, 14, 7, 1, 12, 5, 15, 2, 11, 6, 13, 4, 10, 8, 0}; + constexpr std::array NULLS {}; + auto typed = typed_int32(VALUES, NULLS); + + HybridSorter sorter; + IColumn::Permutation result; + // limit 2 of 16 rows sits exactly on the threshold and still takes the partial_sort branch. + // Pin the whole prefix that branch must produce rather than only its first entry: the values 0 + // and 1 live at rows 15 and 4. The two branches cannot be told apart through the public API, so + // this pins the contract both of them owe, not the branch itself. + typed->get_permutation(false, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {15, 4})); + + // limit 12 of 16 rows exceeds the threshold, so the whole permutation comes out ordered. + typed->get_permutation(false, 12, 0, sorter, result); + ASSERT_EQ(result.size(), VALUES.size()); + for (size_t index = 1; index < result.size(); ++index) { + EXPECT_LT(typed->compare_at(result[index - 1], result[index], *typed, 1), 0); + } +} + TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { auto encoded = ColumnVariantV2::create(); insert_encoded_field(*encoded, encode_json("1")); @@ -3430,19 +3884,90 @@ TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { EXPECT_EQ(json_at(*typed, 2), "{}"); } -TEST(ColumnVariantV2Test, EncodedPhysicalAndOrderingInterfacesStayUnsupported) { +TEST(ColumnVariantV2Test, EncodedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { auto column = ColumnVariantV2::create(); - auto source = ColumnVariantV2::create(); - insert_encoded_field(*source, encode_json("1")); + insert_encoded_field(*column, encode_json(R"("text")")); + insert_encoded_field(*column, encode_json("2")); + insert_encoded_field(*column, encode_json("1.5")); + insert_encoded_field(*column, encode_json("true")); expect_not_implemented([&] { static_cast(column->get_data_at(0)); }, "intentionally unsupported"); HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { column->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); - expect_not_implemented([&] { column->replace_column_data(*source, 0); }, + column->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {3, 2, 1, 0})); + expect_not_implemented([&] { column->replace_column_data(*column, 0); }, "intentionally unsupported"); } +TEST(ColumnVariantV2Test, NullableMultiKeyOrderingWorks) { + auto variant = ColumnVariantV2::create(); + insert_encoded_field(*variant, encode_json("2")); + insert_encoded_field(*variant, encode_json("1")); + insert_encoded_field(*variant, encode_json("1")); + auto nullable = ColumnNullable::create(std::move(variant), ColumnUInt8::create(3, 0)); + + auto tie = ColumnInt32::create(); + tie->insert_value(0); + tie->insert_value(2); + tie->insert_value(1); + auto ids = ColumnInt32::create(); + ids->insert_value(1); + ids->insert_value(2); + ids->insert_value(3); + + Block source; + source.insert({std::move(nullable), make_nullable(std::make_shared()), "v"}); + source.insert({std::move(tie), std::make_shared(), "tie"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, 1, 1); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + + const auto& sorted_ids = assert_cast(*sorted.get_by_position(2).column); + EXPECT_EQ(sorted_ids.get_data()[0], 3); + EXPECT_EQ(sorted_ids.get_data()[1], 2); + EXPECT_EQ(sorted_ids.get_data()[2], 1); +} + +TEST(ColumnVariantV2Test, NullableTypedSortMatchesEncoded) { + constexpr std::array VALUES {3, 0, 1, 7, 1}; + // Row 1 is a Variant null inside the value; row 2 is a SQL NULL in the outer column. + constexpr std::array VARIANT_NULLS {0, 1, 0, 0, 0}; + constexpr std::array SQL_NULLS {0, 0, 1, 0, 0}; + auto sorted_ids = [&](const ColumnPtr& variant, int direction, int nulls_direction) { + auto nulls = ColumnUInt8::create(); + nulls->get_data().assign(SQL_NULLS.begin(), SQL_NULLS.end()); + auto ids = ColumnInt32::create(); + for (int32_t id = 0; id < static_cast(VALUES.size()); ++id) { + ids->insert_value(id); + } + Block source; + source.insert({ColumnNullable::create(variant, std::move(nulls)), + make_nullable(std::make_shared()), "v"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, direction, nulls_direction); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + const auto& output = assert_cast(*sorted.get_by_position(1).column); + return std::vector(output.get_data().begin(), output.get_data().end()); + }; + auto typed = typed_int32(VALUES, VARIANT_NULLS); + auto encoded = encoded_copy_of(*typed); + for (int direction : {-1, 1}) { + for (int nulls_direction : {-1, 1}) { + EXPECT_EQ(sorted_ids(typed->get_ptr(), direction, nulls_direction), + sorted_ids(encoded->get_ptr(), direction, nulls_direction)) + << "direction=" << direction << " nulls_direction=" << nulls_direction; + } + } +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp index 1b5dddf8e9e925..5cecf9d9cefa00 100644 --- a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp @@ -15,27 +15,35 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include +#include #include #include #include #include +#include #include "core/arena.h" #include "core/assert_cast.h" +#include "core/column/column_array.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_number.h" +#include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_variant_v2_serde.h" #include "core/string_buffer.hpp" +#include "core/value/variant/variant_batch_builder.h" +#include "exprs/function/parse/variant_string_parse.h" #include "util/jsonb_document.h" +#include "util/jsonb_utils.h" #include "util/jsonb_writer.h" namespace doris { @@ -103,6 +111,33 @@ std::string json_at(const DataTypeVariantV2SerDe& serde, const IColumn& column, return output->get_data_at(0).to_string(); } +ColumnVariantV2::MutablePtr encoded_json(std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (std::string_view row : rows) { + encoder.add_json({row.data(), row.size()}); + } + VariantBatchBuilder block = encoder.finish_batch(); + auto result = ColumnVariantV2::create(); + result->insert_encoded_batch(block); + return result; +} + +std::vector jsonb_vector_as_json( + const DataTypeSerDe& serde, const IColumn& column, + const DataTypeSerDe::FormatOptions& options = DataTypeSerDe::FormatOptions {}) { + auto output = ColumnString::create(); + const Status status = serde.serialize_column_to_jsonb_vector(column, *output, options); + EXPECT_TRUE(status.ok()) << status; + std::vector rows; + for (size_t row = 0; row < output->size(); ++row) { + const StringRef bytes = output->get_data_at(row); + rows.push_back(JsonbToJson::jsonb_to_json_string(bytes.data, bytes.size)); + } + return rows; +} + std::string object_with_value(const std::function& write_value) { JsonbWriter writer; EXPECT_TRUE(writer.writeStartObject()); @@ -114,6 +149,51 @@ std::string object_with_value(const std::function& write_val } // namespace +TEST(DataTypeVariantV2SerdeJsonbTest, VectorSkipsMaskedTypedNonFiniteValues) { + auto values = ColumnFloat64::create(); + values->insert_value(1.5); + values->insert_value(std::numeric_limits::quiet_NaN()); + values->insert_value(2.5); + auto column = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(values), ColumnUInt8::create(3, 0)), + std::make_shared()); + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + auto output = ColumnString::create(); + const std::array null_map {0, 1, 0}; + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), "1.5"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), "2.5"); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, VectorPreservesMaskedEncodedAndConstantRowPositions) { + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + const std::array null_map {0, 1, 0}; + auto encoded = encoded_json({"1", "null", "2"}); + auto constant = ColumnConst::create(typed_int(42), 3); + for (const IColumn* column : {static_cast(encoded.get()), + static_cast(constant.get())}) { + auto output = ColumnString::create(); + ASSERT_TRUE( + serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), + column == encoded.get() ? "1" : "42"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), + column == encoded.get() ? "2" : "42"); + } +} + TEST(DataTypeVariantV2SerdeJsonbTest, EncodedAndTypedWriteIdenticalBinaryDocumentsAndRoundTrip) { DataTypeVariantV2SerDe serde; auto typed = typed_int(42); @@ -243,4 +323,80 @@ TEST(DataTypeVariantV2SerdeJsonbTest, InvalidRowDoesNotTouchResultWriter) { EXPECT_EQ(writer.getOutput()->getSize(), before); } +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbWritesTheJsonDocument) { + DataTypeVariantV2SerDe serde; + auto encoded = encoded_json({R"({"b":[true,null,"x"],"a":{"c":1.5}})", "42", R"("s")", "null"}); + EXPECT_EQ(jsonb_vector_as_json(serde, *encoded), + (std::vector {R"({"a":{"c":1.5},"b":[true,null,"x"]})", "42", R"("s")", + "null"})); + + // Typed and encoded columns write the same bytes, including a Variant null. + for (const bool is_null : {false, true}) { + auto typed = typed_int(42, is_null); + ColumnPtr encoded_int = encoded_copy(*typed); + auto typed_output = ColumnString::create(); + auto encoded_output = ColumnString::create(); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*typed, *typed_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*encoded_int, *encoded_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_EQ(typed_output->size(), 1); + EXPECT_EQ(typed_output->get_data_at(0), encoded_output->get_data_at(0)); + EXPECT_EQ(jsonb_vector_as_json(serde, *typed)[0], is_null ? "null" : "42"); + } + + ColumnPtr constant = ColumnConst::create(encoded_json({R"({"k":"v"})"})->get_ptr(), 2); + EXPECT_EQ(jsonb_vector_as_json(serde, *constant), + (std::vector {R"({"k":"v"})", R"({"k":"v"})"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbNestsVariantInArray) { + auto values = encoded_json({R"({"a":[1,2]})", "7", R"("x")"}); + auto value_nulls = ColumnUInt8::create(); + for (const uint8_t is_null : {0, 1, 0}) { + value_nulls->insert_value(is_null); + } + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(2); + offsets->insert_value(3); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), std::move(value_nulls)), std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array), + (std::vector {R"([{"a":[1,2]},null])", R"(["x"])"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbUsesTheSessionTimeZone) { + VariantBatchBuilder builder; + auto row = builder.begin_row(); + row.add_timestamp_micros(0, true); + row.finish(); + VariantBatchBuilder block = builder.finish_batch(); + auto values = ColumnVariantV2::create(); + values->insert_encoded_batch(block); + + DataTypeVariantV2SerDe serde; + const cctz::time_zone shanghai = cctz::fixed_time_zone(cctz::seconds(8 * 3600)); + DataTypeSerDe::FormatOptions options; + options.timezone = &shanghai; + EXPECT_EQ(jsonb_vector_as_json(serde, *values, options), + (std::vector {R"("1970-01-01 08:00:00.000000+08:00")"})); + EXPECT_EQ(jsonb_vector_as_json(serde, *values), + (std::vector {R"("1970-01-01 00:00:00.000000+00:00")"})); + + // A nested Variant element receives the same time zone. + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), ColumnUInt8::create(1, 0)), + std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array, options), + (std::vector {R"(["1970-01-01 08:00:00.000000+08:00"])"})); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp index 2972e7023da4ae..b8b1c598ae0261 100644 --- a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp @@ -53,7 +53,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt32::create(); @@ -65,7 +66,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt64::create(); EXPECT_TRUE(data.type->get_serde()->deserialize_column_from_jsonb( @@ -77,7 +79,8 @@ TEST(DataTypeWritToJsonb, test_number) { auto data = ColumnHelper::create_column_with_name( {1.1F, 2.2F, 3.3F, 4.4F, 5.5F}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4.4"); auto data_to = ColumnFloat32::create(); @@ -92,7 +95,8 @@ TEST(DataTypeWritToJsonb, test_number) { col->insert_value(val); auto type = std::make_shared(18, 2); JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer, + DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "123.45"); auto data_to = ColumnDecimal128V3::create(0, 2); @@ -109,7 +113,8 @@ TEST(DataTypeWritToJsonb, test_string) { auto data = ColumnHelper::create_column_with_name( {"hello", "world", "doris", "vectorized", "test"}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "\"vectorized\""); auto data_to = ColumnString::create(); @@ -137,7 +142,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[1,2,3]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -148,7 +154,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[null,5]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -182,7 +189,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":1,\"string_col\":\"hello\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -194,7 +202,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":2,\"string_col\":\"world\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -206,7 +215,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 2, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 2, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":3,\"string_col\":null}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -217,7 +227,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 3, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":null,\"string_col\":\"vectorized\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -229,7 +240,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 4, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 4, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":5,\"string_col\":\"test\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( diff --git a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp index 54bedaa5c52cf0..5ed330f46425d1 100644 --- a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp +++ b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp @@ -95,6 +95,43 @@ TEST(CastVariantV2ToTest, UnsupportedSourceReturnsAnError) { EXPECT_EQ(cast.column.get(), cast.initial_result.get()); } +TEST(CastVariantV2ToTest, InvalidUtf8StringSourceReturnsAnError) { + const std::array source_types { + std::make_shared(), std::make_shared(16, TYPE_CHAR), + std::make_shared(64, TYPE_VARCHAR)}; + for (const DataTypePtr& source_type : source_types) { + auto source = ColumnString::create(); + source->insert_data("ok", 2); + source->insert_data("\xFF", 1); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) + << source_type->get_name() << ": " << cast.status; + + // A row that the cast turns into NULL never becomes a Variant string. + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << source_type->get_name() << ": " << null_row.status; + } +} + +TEST(CastVariantV2ToTest, InvalidDateSourceReturnsAnError) { + // The same check covers every scalar Variant cannot hold, not only strings. + auto source = ColumnDateV2::create(); + DateV2Value valid; + valid.unchecked_set_time(2024, 1, 2, 0, 0, 0); + DateV2Value invalid; + ASSERT_FALSE(invalid.is_valid_date()); + source->insert_value(valid); + source->insert_value(invalid); + const auto source_type = std::make_shared(); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) << cast.status; + + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << null_row.status; +} + TEST(CastVariantV2ToTest, Decimal256ReturnsAnErrorInsteadOfStringifying) { auto type = std::make_shared(76, 2); auto source = ColumnDecimal256::create(0, 2); diff --git a/be/test/exprs/variant_v2_execution_test.cpp b/be/test/exprs/variant_v2_execution_test.cpp index 3aed252297870d..36fd2e1bec95df 100644 --- a/be/test/exprs/variant_v2_execution_test.cpp +++ b/be/test/exprs/variant_v2_execution_test.cpp @@ -19,8 +19,12 @@ #include "core/column/column_variant.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" +#include "exprs/function/functions_comparison.h" +#include "exprs/function/parse/variant_string_parse.h" namespace doris { @@ -32,4 +36,79 @@ TEST(VariantV2ExecutionTest, ExecutionTypeSelectsPhysicalColumn) { EXPECT_NE(check_and_get_column(*compute_v2.create_column()), nullptr); } +TEST(VariantV2ExecutionTest, TypePropertiesDoNotAffectEquality) { + // Execution compares and hashes Variant values canonically; storage properties such as the + // maximum subcolumn count and doc mode must not make two Variant types different. + const DataTypePtr plain = std::make_shared(2048, false); + const DataTypePtr limited = std::make_shared(10, false); + const DataTypePtr doc_mode = std::make_shared(0, true); + EXPECT_TRUE(plain->equals(*limited)); + EXPECT_TRUE(limited->equals(*doc_mode)); + EXPECT_TRUE(make_nullable(plain)->equals(*make_nullable(doc_mode))); + EXPECT_FALSE(plain->equals(DataTypeInt32())); +} + +TEST(VariantV2ExecutionTest, CanonicalComparison) { + auto parse = [](std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 255, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (auto json : rows) { + encoder.add_json({json.data(), json.size()}); + } + auto column = ColumnVariantV2::create(); + column->insert_encoded_batch(encoder.finish_batch()); + return column; + }; + auto left = parse({"1", "1", "null", R"({"a":1,"b":[2,null]})", "[1,2]", "true"}); + auto right = parse({"1.0", R"("1")", "null", R"({"b":[2.0,null],"a":1.0})", "[2,1]", "1"}); + auto type = std::make_shared(); + Block block {{left->get_ptr(), type, "left"}, + {right->get_ptr(), type, "right"}, + {nullptr, std::make_shared(), "result"}}; + FunctionComparison equals; + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 0, 1, 1, 0, 0})); + FunctionComparison not_equals; + ASSERT_TRUE(not_equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + FunctionComparison less; + ASSERT_TRUE(less.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + + auto one = parse({"1.0"}); + block.get_by_position(1).column = ColumnConst::create(std::move(one), left->size()); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {1, 0}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); +} + +TEST(VariantV2ExecutionTest, TypedEquality) { + auto numbers = ColumnInt32::create(); + numbers->get_data() = {1, 2, 3}; + auto nulls = ColumnUInt8::create(); + nulls->get_data() = {0, 0, 1}; + auto typed = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(numbers), std::move(nulls)), + std::make_shared()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(*typed, 0, typed->size()); + encoded->ensure_encoded(); + ASSERT_TRUE(typed->is_typed()); + ASSERT_FALSE(encoded->is_typed()); + for (const auto& left : {typed->get_ptr(), encoded->get_ptr()}) { + for (const auto& right : {typed->get_ptr(), encoded->get_ptr()}) { + for (size_t row = 0; row < left->size(); ++row) { + EXPECT_EQ(left->compare_at(row, row, *right, 1), 0); + } + } + } +} + } // namespace doris diff --git a/be/test/util/variant/variant_canonical_test.cpp b/be/test/util/variant/variant_canonical_test.cpp index 42662c7262cd1b..83f2189494f16c 100644 --- a/be/test/util/variant/variant_canonical_test.cpp +++ b/be/test/util/variant/variant_canonical_test.cpp @@ -384,6 +384,7 @@ void expect_scalar_equivalent(const VariantScalarRef& scalar_ref, const OwnedVal EXPECT_EQ(scalar_hashes.crc, encoded_hashes.crc); EXPECT_EQ(scalar_hashes.crc32c, encoded_hashes.crc32c); EXPECT_EQ(arena(scalar_ref), arena(encoded_value.ref())); + EXPECT_EQ(canonical_compare(scalar_ref, encoded_value.ref()), 0); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive typed/encoded parity matrix. @@ -548,6 +549,119 @@ TEST(VariantCanonicalTest, NumericEquivalenceMatrix) { VariantPrimitiveId::DOUBLE); } +TEST(VariantCanonicalTest, ComparisonPreservesEqualityAndNumericOrder) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_FALSE(canonical_equals(left.ref(), right.ref())); + }; + const auto expect_equal = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_EQ(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_EQ(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_TRUE(canonical_equals(left.ref(), right.ref())); + EXPECT_EQ(hashes(left.ref()).sip, hashes(right.ref()).sip); + }; + + expect_order(decimal_value(15, 1), integer_value(100, 1)); + expect_order(decimal_value(1, 1), double_value(0.1)); + expect_order(double_value(std::nextafter(1.0, 0.0)), integer_value(1, 1)); + expect_order(integer_value(1, 1), double_value(std::nextafter(1.0, 2.0))); + expect_order(decimal_value(15, 1), double_value(1.5)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-999999999999999999LL, 1)); + expect_order(double_value(std::numeric_limits::infinity()), + double_value(std::numeric_limits::quiet_NaN())); + + expect_equal(integer_value(42, 1), decimal_value(4200, 2)); + expect_equal(integer_value(42, 1), double_value(42.0)); + expect_equal(double_bits(0x7FF0000000000001ULL), double_bits(0xFFFABCDE12345678ULL)); +} + +TEST(VariantCanonicalTest, ContainerComparisonIsLexicographicAndTransitive) { + const OwnedValue array_a = array_value({integer_value(1, 1).value}); + const OwnedValue array_b = array_value({integer_value(1, 1).value, integer_value(2, 1).value}); + const OwnedValue array_c = array_value({integer_value(1, 1).value, integer_value(3, 1).value}); + EXPECT_LT(canonical_compare(array_a.ref(), array_b.ref()), 0); + EXPECT_LT(canonical_compare(array_b.ref(), array_c.ref()), 0); + EXPECT_LT(canonical_compare(array_a.ref(), array_c.ref()), 0); + + const OwnedValue object_a = object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0}); + const OwnedValue object_b = object_value({"a"}, true, {0}, {integer_value(2, 1).value}, {0}); + const OwnedValue object_c = object_value({"b"}, true, {0}, {integer_value(0, 1).value}, {0}); + EXPECT_LT(canonical_compare(object_a.ref(), object_b.ref()), 0); + EXPECT_LT(canonical_compare(object_b.ref(), object_c.ref()), 0); + EXPECT_LT(canonical_compare(object_a.ref(), object_c.ref()), 0); + EXPECT_GT(canonical_compare(object_c.ref(), object_a.ref()), 0); +} + +TEST(VariantCanonicalTest, ComparisonBoundariesAndCrossKindTransitivity) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + }; + __int128 max_38_digits = 1; + for (int digit = 0; digit < 38; ++digit) { + max_38_digits *= 10; + } + max_38_digits -= 1; + + // Scaling 38-digit magnitudes by 10^38 needs the 256-bit intermediate. + expect_order(decimal_value(max_38_digits, 38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(-max_38_digits, 0), decimal_value(-max_38_digits, 38)); + // Subnormal and large doubles against exact values at the decimal limits. + expect_order(double_value(std::numeric_limits::denorm_min()), decimal_value(1, 38)); + expect_order(double_value(1e38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), double_value(1.7e38)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), + double_value(std::numeric_limits::infinity())); + + // Numeric value first, canonical kind as the tie-breaker, must stay transitive. + const std::array chain {decimal_value(15, 1), double_value(1.5), + integer_value(2, 1), decimal_value(25, 1)}; + for (size_t left = 0; left < chain.size(); ++left) { + for (size_t right = left + 1; right < chain.size(); ++right) { + expect_order(chain[left], chain[right]); + } + } + // Numbers order before strings and containers. + expect_order(double_value(std::numeric_limits::quiet_NaN()), string_value("0")); + expect_order(string_value("0"), array_value({})); +} + +TEST(VariantCanonicalTest, ScalarAgainstEncodedRootMatchesEncodedComparison) { + // The mixed overload must order a scalar exactly as its encoded bytes would be ordered, for + // every kind of encoded root including the containers a scalar can never equal. + const std::string text = "b"; + const std::array scalars { + VariantScalarRef::null_value(), + VariantScalarRef::boolean(true), + VariantScalarRef::integer(2), + VariantScalarRef::decimal(15, 1), + VariantScalarRef::float64(1.5), + VariantScalarRef::float64(std::numeric_limits::quiet_NaN()), + VariantScalarRef::string(StringRef(text)), + VariantScalarRef::date(19000)}; + const std::array roots { + scalar(primitive(VariantPrimitiveId::NULL_VALUE)), + integer_value(2, 1), + decimal_value(15, 1), + double_value(1.5), + double_value(2.5), + string_value("a"), + string_value("b"), + array_value({integer_value(1, 1).value}), + object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0})}; + for (const VariantScalarRef& value : scalars) { + const OwnedValue encoded = scalar(physical(value)); + for (const OwnedValue& root : roots) { + EXPECT_EQ(canonical_compare(value, root.ref()), + canonical_compare(encoded.ref(), root.ref())); + } + } +} + TEST(VariantCanonicalTest, PrimitiveTypeClasses) { const OwnedValue null_a = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); const OwnedValue null_b = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); @@ -881,6 +995,11 @@ TEST(VariantCanonicalTest, RandomizedEqualsHashAndArenaProperties) { auto [left, right] = random_pair(random, index, expected_equal); const bool equal = canonical_equals(left.ref(), right.ref()); EXPECT_EQ(equal, expected_equal); + const int comparison = canonical_compare(left.ref(), right.ref()); + const int reverse_comparison = canonical_compare(right.ref(), left.ref()); + EXPECT_EQ(comparison == 0, equal); + EXPECT_EQ((comparison > 0) - (comparison < 0), + -((reverse_comparison > 0) - (reverse_comparison < 0))); const std::string left_arena = arena(left.ref()); const std::string right_arena = arena(right.ref()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index 0acf44f3c2ec7e..f7fb2205f7472e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -26,6 +26,14 @@ public class TypeTest { + @Test + public void testTimestampTzIsNotVariantSubtype() { + Assert.assertFalse(Type.VARIANT.supportSubType(ScalarType.createTimeStampTzType(6))); + Assert.assertFalse(Type.VARIANT.supportSubType( + new ArrayType(ScalarType.createTimeStampTzType(3), true))); + Assert.assertTrue(Type.VARIANT.supportSubType(ScalarType.createDatetimeV2Type(6))); + } + // ===================== ArrayType ===================== @Test public void testArrayOfArrayExactMatch() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java index b5beafa169356b..f326d74fb73325 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; @@ -276,6 +277,18 @@ private String evidence() { } } + @Test + public void testTargetVariantTypeParticipatesInTryParseEquality() { + StringLiteral json = new StringLiteral("{\"a\":1}"); + VariantType firstTarget = new VariantType(1); + VariantType secondTarget = new VariantType(2); + + TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); + TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); + Assertions.assertNotEquals(firstTryParse, secondTryParse); + Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); + } + @FunctionalInterface private interface LoadFixtureSupplier { LoadFixture get() throws Exception; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java index bd8a8af9bc3401..bc8d3c8ef0e54f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java @@ -29,11 +29,13 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; +import org.apache.doris.nereids.processor.post.RuntimeFilterPushDownVisitor.PushDownContext; import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -57,6 +59,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.RuntimeFilterId; @@ -88,6 +91,10 @@ public void runBeforeAll() throws Exception { connectContext.getSessionVariable().setEnableRuntimeFilterPrune(false); connectContext.getSessionVariable().expandRuntimeFilterByInnerJoin = false; connectContext.getSessionVariable().setDisableJoinReorder(true); + createTable("CREATE TABLE variant_rf_a (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); + createTable("CREATE TABLE variant_rf_b (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); } @Test @@ -592,4 +599,91 @@ public void testRuntimeFilterBlockByRecCte() { List filters = context.getNereidsRuntimeFilter(); Assertions.assertEquals(0, filters.size()); } + + @Test + public void testVariantJoinKeyDoesNotGenerateRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + // Control: the INT key of the same tables produces a runtime filter in this setup. + Assertions.assertFalse(getRuntimeFilters( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.k = b.k").get().isEmpty()); + // Scalar runtime filters cannot express Variant canonical equality. + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v = b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v <=> b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v['id'] = b.v['id']", + "SELECT * FROM (SELECT CAST(k AS VARIANT) vv FROM variant_rf_a) x " + + "JOIN variant_rf_b b ON x.vv = b.v")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testVariantPushDownContextIsInvalidBeforeTraversal() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + RuntimeFilterContext context = new RuntimeFilterContext( + connectContext.getSessionVariable(), RuntimeFilterId.createGenerator()); + AbstractPhysicalJoin builder = Mockito.mock(AbstractPhysicalJoin.class); + SlotReference scalar = new SlotReference("scalar", IntegerType.INSTANCE); + SlotReference variant = new SlotReference("variant", VariantType.INSTANCE); + Expression cast = new Cast(variant, IntegerType.INSTANCE); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + context.aliasTransferMapPut(scalar, Pair.of(scan, scalar)); + context.aliasTransferMapPut(variant, Pair.of(scan, variant)); + for (TRuntimeFilterType type : ImmutableList.of( + TRuntimeFilterType.IN, TRuntimeFilterType.BLOOM, + TRuntimeFilterType.MIN_MAX, TRuntimeFilterType.IN_OR_BLOOM)) { + Assertions.assertFalse(PushDownContext.createPushDownContextForHashJoin( + variant, scalar, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertFalse(PushDownContext.createPushDownContextForHashJoin( + scalar, variant, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContextForHashJoin( + scalar, cast, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContextForHashJoin( + cast, scalar, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testScalarCastOfVariantGeneratesRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b " + + "ON CAST(a.v AS INT) = CAST(b.v AS INT)", + "SELECT * FROM (SELECT CAST(v AS INT) vv FROM variant_rf_a) a " + + "JOIN variant_rf_b b ON a.vv = b.k")) { + Assertions.assertFalse(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testVariantSetOperationDoesNotGenerateRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + for (String sql : ImmutableList.of( + "SELECT v FROM variant_rf_a INTERSECT SELECT v FROM variant_rf_b", + "SELECT v FROM variant_rf_a EXCEPT SELECT v FROM variant_rf_b")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + Assertions.assertTrue(getSetOperationRuntimeFilters(sql).isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java deleted file mode 100644 index 6598da11c6c36a..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.analysis; - -import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.types.VariantType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ExpressionAnalyzerVariantAutoCastTest { - - @Test - public void testTargetVariantTypeParticipatesInTryParseEquality() { - StringLiteral json = new StringLiteral("{\"a\":1}"); - VariantType firstTarget = new VariantType(1); - VariantType secondTarget = new VariantType(2); - - TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); - TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); - Assertions.assertNotEquals(firstTryParse, secondTryParse); - Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java index e167821dea2e60..d3d2dcea5e5696 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java @@ -20,9 +20,12 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.thrift.TUniqueId; import org.apache.doris.utframe.TestWithFeService; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; @@ -30,6 +33,8 @@ class VariantEqualityContextTest extends TestWithFeService { + private boolean originalEnableVariantV2; + @Override protected void runBeforeAll() throws Exception { createDatabase("variant_equality_context_test"); @@ -49,6 +54,10 @@ void testLegacyVariantBehaviorIsPreservedWhenV2IsDisabled() { Config.enable_variant_v2 = false; assertRejected("SELECT v, COUNT(*) FROM t1 GROUP BY v", "Doris hll, bitmap"); + assertCheckRejected("SELECT v FROM t1 ORDER BY v", "Doris hll, bitmap"); + assertCheckRejected("SELECT v FROM t1 ORDER BY v LIMIT 10", "Doris hll, bitmap"); + assertRejected("SELECT row_number() OVER (ORDER BY v) FROM t1", "Doris hll, bitmap"); + assertRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", "could not used in ComparisonPredicate"); assertRejected("SELECT DISTINCT v FROM t1", "Doris hll, bitmap"); assertRejected("SELECT COUNT(DISTINCT v) FROM t1", "COUNT DISTINCT"); assertRejected("SELECT v FROM t1 INTERSECT SELECT v FROM t2", "Doris hll, bitmap"); @@ -68,30 +77,71 @@ void testLegacyVariantBehaviorIsPreservedWhenV2IsDisabled() { } } - @Test - void testVariantV2CanonicalHashContextsAndComparisonRestrictions() { - boolean originalEnableVariantV2 = Config.enable_variant_v2; + @BeforeEach + void enableVariantV2() { + originalEnableVariantV2 = Config.enable_variant_v2; Config.enable_variant_v2 = true; - try { - assertAllAccepted( - "SELECT parse_to_variant(CAST(k AS STRING)), COUNT(*) FROM t1 " - + "GROUP BY parse_to_variant(CAST(k AS STRING))", - "SELECT DISTINCT parse_to_variant(CAST(k AS STRING)) FROM t1", - "SELECT COUNT(DISTINCT parse_to_variant(CAST(k AS STRING))) FROM t1", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 INTERSECT " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 EXCEPT " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 UNION " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2"); - - assertVariantComparisonRejected("SELECT parse_to_variant('1') = parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant('1') != parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant('1') <=> parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant(CAST(k AS STRING)) = k FROM t1"); - } finally { - Config.enable_variant_v2 = originalEnableVariantV2; - } + } + + @AfterEach + void restoreVariantV2() { + Config.enable_variant_v2 = originalEnableVariantV2; + } + + @Test + void testRejectedVariantOrderingAndMixedComparisons() { + assertVariantComparisonRejected("SELECT v = k FROM t1"); + assertVariantComparisonRejected("SELECT v > v FROM t1"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); + } + + @Test + void testVariantEquality() { + assertAllAccepted( + "SELECT v = v, v != v, v <=> v FROM t1", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", + "SELECT * FROM t1 LEFT JOIN t2 ON t1.v <=> t2.v", + "SELECT * FROM t1 FULL JOIN t2 ON t1.v = t2.v AND t1.k = t2.k", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v OR t1.k = t2.k", + "SELECT EXISTS(SELECT 1 FROM t2 WHERE t1.v = t2.v) FROM t1", + "SELECT * FROM t1 WHERE v IN (SELECT v FROM t2)", + "SELECT * FROM t1 WHERE v NOT IN (SELECT v FROM t2)", + "SELECT * FROM t1 JOIN t2 ON t1.v['id'] = t2.v['id']"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.k"); + } + + @Test + void testLegacyVariantEqualityWhenDefaultIsDisabled() { + Config.enable_variant_v2 = false; + assertRejected("SELECT parse_to_variant('1') = parse_to_variant('1.0')", + "could not used in ComparisonPredicate"); + assertRejected("SELECT parse_to_variant('1') <=> NULL", + "could not used in ComparisonPredicate"); + assertRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", + "could not used in ComparisonPredicate"); + } + + @Test + void testVariantV2CanonicalHashContexts() { + assertAllAccepted( + "SELECT parse_to_variant(CAST(k AS STRING)), COUNT(*) FROM t1 " + + "GROUP BY parse_to_variant(CAST(k AS STRING))", + "SELECT DISTINCT parse_to_variant(CAST(k AS STRING)) FROM t1", + "SELECT COUNT(DISTINCT parse_to_variant(CAST(k AS STRING))) FROM t1", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 INTERSECT " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 EXCEPT " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 UNION " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2"); + } + + @Test + void testVariantOrdering() { + assertPlanAccepted("SELECT v FROM t1 ORDER BY v"); + assertPlanAccepted("SELECT v FROM t1 ORDER BY v LIMIT 10"); + assertPlanAccepted("SELECT row_number() OVER (ORDER BY v) FROM t1"); } @Test @@ -114,6 +164,17 @@ private void assertAllAccepted(String... sqlStatements) { .map(sql -> (Executable) () -> assertAccepted(sql))); } + private void assertPlanAccepted(String sql) { + connectContext.setQueryId(new TUniqueId(1, 1)); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).plan(sql), sql); + } + + private void assertCheckRejected(String sql, String expectedMessage) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze(sql).applyBottomUp(new CheckAfterRewrite()), sql); + Assertions.assertTrue(exception.getMessage().contains(expectedMessage), exception.getMessage()); + } + private void assertVariantComparisonRejected(String sql) { assertRejected(sql, "CAST to a concrete type first"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index 1f0e6722dc03a0..3fc689f6c289f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -17,10 +17,12 @@ package org.apache.doris.nereids.util; +import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -36,6 +38,8 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArray; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToJson; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; @@ -141,30 +145,29 @@ public void testImplicitCastFromPrimitiveToString() { } @Test - public void testVariantV2ToJsonImplicitCastRequiresExplicitCast() { - boolean originalEnableVariantV2 = Config.enable_variant_v2; - try { - Config.enable_variant_v2 = false; - Assertions.assertEquals(JsonType.INSTANCE, TypeCoercionUtils.implicitCast( - VariantType.INSTANCE, JsonType.INSTANCE).get()); - Config.enable_variant_v2 = true; - Assertions.assertFalse(TypeCoercionUtils.implicitCast( - VariantType.INSTANCE, JsonType.INSTANCE).isPresent()); - } finally { - Config.enable_variant_v2 = originalEnableVariantV2; - } + public void testVariantToJsonImplicitCast() { + Assertions.assertEquals(JsonType.INSTANCE, + TypeCoercionUtils.implicitCast(VariantType.INSTANCE, JsonType.INSTANCE).get()); + } + + @Test + public void testVariantToJsonFunctionSignature() { + Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( + JsonType.INSTANCE, VariantType.INSTANCE)); } @Test - public void testVariantV2ToJsonFunctionSignatureRequiresExplicitCast() { + public void testToJsonAcceptsVariantArgument() { boolean originalEnableVariantV2 = Config.enable_variant_v2; + Config.enable_variant_v2 = true; try { - Config.enable_variant_v2 = false; - Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( - JsonType.INSTANCE, VariantType.INSTANCE)); - Config.enable_variant_v2 = true; - Assertions.assertFalse(ExplicitlyCastableSignature.isExplicitlyCastable( - JsonType.INSTANCE, VariantType.INSTANCE)); + SlotReference variant = new SlotReference("v", new VariantType(100)); + FunctionSignature signature = new ToJson(variant).getSignature(); + Assertions.assertEquals(JsonType.INSTANCE, signature.returnType); + Assertions.assertEquals(variant.getDataType(), signature.getArgType(0)); + // JSON builders keep converting a Variant argument through to_json. + Expression array = new JsonArray(variant).rewriteWhenAnalyze(); + Assertions.assertEquals(new ToJson(variant), array.child(0)); } finally { Config.enable_variant_v2 = originalEnableVariantV2; } @@ -403,22 +406,36 @@ public void testProcessComparisonPredicateDowngrade() { } @Test - public void testVariantV2ComparisonRequiresExplicitCast() { + public void testVariantComparisonRequiresExplicitCast() { boolean originalEnableVariantV2 = Config.enable_variant_v2; Config.enable_variant_v2 = true; - SlotReference variant = new SlotReference("v", VariantType.INSTANCE); - SlotReference anotherVariant = new SlotReference("v2", VariantType.INSTANCE); - SlotReference integer = new SlotReference("i", IntegerType.INSTANCE); - ElementAt variantSubpath = new ElementAt(variant, new StringLiteral("c")); - ElementAt anotherVariantSubpath = new ElementAt(anotherVariant, new StringLiteral("c")); try { - AnalysisException equality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(variant, anotherVariant))); - Assertions.assertTrue(equality.getMessage().contains("CAST to a concrete type first")); - - AnalysisException nullSafeEquality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new NullSafeEqual(variant, anotherVariant))); - Assertions.assertTrue(nullSafeEquality.getMessage().contains("CAST to a concrete type first")); + SlotReference variant = new SlotReference("v", VariantType.INSTANCE); + SlotReference anotherVariant = new SlotReference("v2", VariantType.INSTANCE); + SlotReference integer = new SlotReference("i", IntegerType.INSTANCE); + ElementAt variantSubpath = new ElementAt(variant, new StringLiteral("c")); + ElementAt anotherVariantSubpath = new ElementAt(anotherVariant, new StringLiteral("c")); + + // Variant equality uses canonical comparison, so it is accepted without a cast. + EqualTo equality = new EqualTo(variant, anotherVariant); + Assertions.assertSame(equality, TypeCoercionUtils.processComparisonPredicate(equality)); + + NullSafeEqual nullSafeEquality = new NullSafeEqual(variant, anotherVariant); + Assertions.assertSame(nullSafeEquality, TypeCoercionUtils.processComparisonPredicate(nullSafeEquality)); + + // A bare NULL takes the Variant type on either side; an ordering comparison stays rejected. + for (ComparisonPredicate bareNull : ImmutableList.of( + new EqualTo(variant, NullLiteral.INSTANCE), new EqualTo(NullLiteral.INSTANCE, variant), + new NullSafeEqual(variant, NullLiteral.INSTANCE), + new NullSafeEqual(NullLiteral.INSTANCE, variant))) { + Expression coerced = TypeCoercionUtils.processComparisonPredicate(bareNull); + Assertions.assertEquals(bareNull.getClass(), coerced.getClass()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(0).getDataType()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(1).getDataType()); + } + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate( + new GreaterThan(variant, NullLiteral.INSTANCE))); AnalysisException mixedType = Assertions.assertThrows(AnalysisException.class, () -> TypeCoercionUtils.processComparisonPredicate(new GreaterThan(variant, integer))); @@ -439,18 +456,11 @@ public void testVariantV2ComparisonRequiresExplicitCast() { Assertions.assertEquals(subpathComparison.child(0).getDataType(), subpathComparison.child(1).getDataType()); - Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate( - new EqualTo(variantSubpath, anotherVariantSubpath))); + EqualTo subpathEquality = new EqualTo(variantSubpath, anotherVariantSubpath); + Assertions.assertSame(subpathEquality, TypeCoercionUtils.processComparisonPredicate(subpathEquality)); Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( new GreaterThan(new Cast(variant, IntegerType.INSTANCE), integer))); - - Config.enable_variant_v2 = false; - Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( - new EqualTo(variant, integer))); - Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( - new GreaterThan(variant, integer))); } finally { Config.enable_variant_v2 = originalEnableVariantV2; } diff --git a/regression-test/data/nereids_p0/join/test_join_on.out b/regression-test/data/nereids_p0/join/test_join_on.out index b14a14dcc75a94..0b96665bbd0451 100644 --- a/regression-test/data/nereids_p0/join/test_join_on.out +++ b/regression-test/data/nereids_p0/join/test_join_on.out @@ -3,3 +3,7 @@ 1 [1, 2] \N \N 2 [2, 3] \N \N +-- !variant_join -- +1 3 +2 4 + diff --git a/regression-test/data/variant_p0/test_variant_bool_path.out b/regression-test/data/variant_p0/test_variant_bool_path.out new file mode 100644 index 00000000000000..affc49e1129fe7 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_bool_path.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_values -- +1 true +2 false +3 true +4 false + +-- !sql_group -- +2 1 +2 2 + +-- !sql_equality -- +1 +3 + +-- !sql_variant_group -- +2 1 +2 2 +1 5 + +-- !sql_rowsets_values_before_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_before_compaction -- +2 1 +2 2 + +-- !sql_rowsets_values_after_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_after_compaction -- +2 1 +2 2 + +-- !sql_buckets_values -- +1 true +2 false +3 true +4 false +5 true +6 false +7 true +8 false + +-- !sql_buckets_group -- +4 1 +4 2 + diff --git a/regression-test/data/variant_p0/test_variant_join.out b/regression-test/data/variant_p0/test_variant_join.out new file mode 100644 index 00000000000000..2ce98242b952e0 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join.out @@ -0,0 +1,160 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !shuffle -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !null_safe -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + +-- !composite_null_safe -- +1 1 + +-- !left -- +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !full -- +\N 4 +\N 6 +\N 9 +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !semi -- +1 +2 +3 +4 +6 +8 +9 + +-- !anti -- +10 +5 +7 + +-- !in -- +1 +2 +3 +4 +6 +8 +9 + +-- !not_in -- + +-- !mark -- +1 true +10 \N +2 true +3 true +4 true +5 \N +6 true +7 \N +8 true +9 true + +-- !exists -- +1 true +10 false +2 true +3 true +4 true +5 false +6 true +7 false +8 true +9 true + +-- !subpath -- +9 8 + +-- !residual -- +1 1 +2 1 +2 2 +3 2 +3 3 +4 3 +4 4 +5 5 +6 5 +6 6 +7 7 +8 7 +8 8 +9 8 +9 9 + +-- !scalar_equality -- +1 true false false +10 false true false +2 true false false +3 false true false +4 false true false +5 \N \N true +6 false true false +7 false true false +8 false true false +9 false true false + +-- !json_null_equality -- +true false + +-- !not_in_non_null -- +10 +7 + +-- !typed_key -- +1 1 + +-- !spill -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + diff --git a/regression-test/data/variant_p0/test_variant_join_type_properties.out b/regression-test/data/variant_p0/test_variant_join_type_properties.out new file mode 100644 index 00000000000000..7ce010d8b2605d --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join_type_properties.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 2 +4 4 + +-- !shuffle -- +1 1 +2 2 +4 4 + +-- !null_safe -- +1 1 +2 2 +3 3 +4 4 + +-- !subpath -- +4 4 + +-- !left_semi -- +1 +2 +4 + +-- !left_anti -- +3 +5 + +-- !mark -- +1 true +2 true +3 \N +4 true +5 \N + +-- !not_in -- +5 + +-- !doc_mode -- +1 1 +4 4 + +-- !computed_key -- +1 1 +1 5 +2 2 +4 4 + +-- !inferred_key -- +1 1 +2 2 + diff --git a/regression-test/data/variant_p0/test_variant_json_functions.out b/regression-test/data/variant_p0/test_variant_json_functions.out new file mode 100644 index 00000000000000..bbf43f503d8849 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_json_functions.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_set_replace_remove -- +1 {"a":1,"b":{"c":[1,2]},"s":"x","z":9} {"a":0,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"r"} {"b":{"c":[1,2]},"s":"x"} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz","z":9} {"a":0,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"r"} {"b":{"c":[3]},"n":{"m":true},"s":"yz"} +6 \N \N \N \N + +-- !extract -- +1 [1,2] x 1 1 1 \N false \N x 1 +2 [3] yz 2 2 2 true false \N yz 2 +3 \N \N \N \N \N \N \N "two" \N \N +4 \N \N \N \N \N \N \N \N \N \N +5 \N \N \N \N \N \N \N \N \N \N +6 \N \N \N \N \N \N \N \N \N \N + +-- !inspect -- +1 ["a", "b", "s"] 3 object true true "$.s" 2600325834747410660 {"a":1,"b.c":[1,2],"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} +2 ["a", "b", "n", "s"] 4 object true false \N 8328850723868929851 {"a":2,"b.c":[3],"n.m":true,"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} +3 \N 3 array false \N \N 9010800628419254104 [1,"two",{"three":3}] [1,"two",{"three":3}] [1,"two",{"three":3}] +4 \N 1 int false \N \N 8321545726482568344 42 42 42 +5 \N 1 string false \N \N -1017189637099138661 "abc" "abc" "abc" +6 \N \N \N \N \N \N \N \N \N \N + +-- !construct -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} [{"a":1,"b":{"c":[1,2]},"s":"x"},1] [{"a":1,"b":{"c":[1,2]},"s":"x"}] {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"b":{"c":[1,2]}} {"v":1} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"},1] [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}] {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"b":{"c":[3]}} {"v":2} +3 [1,"two",{"three":3}] [[1,"two",{"three":3}],1] [[1,"two",{"three":3}]] {"v":[1,"two",{"three":3}]} {"v":[1,"two",{"three":3}]} {"b":null} {"v":null} +4 42 [42,1] [42] {"v":42} {"v":42} {"b":null} {"v":null} +5 "abc" ["abc",1] ["abc"] {"v":"abc"} {"v":"abc"} {"b":null} {"v":null} +6 \N [null,1] [] {"v":null} {"v":null} {"b":null} {"v":null} + +-- !array_of_variant -- +3 [1,"two",{"three":3}] [1,"two",{"three":3}] [[1,"two",{"three":3}]] + +-- !same_as_explicit_cast -- +1 true true true true true true +2 true true true true true true +3 true true true true true true +4 true true true true true true +5 true true true true true true +6 true true true true true true + +-- !string_signature -- +1 1 "{\\"a\\":1,\\"b\\":{\\"c\\":[1,2]},\\"s\\":\\"x\\"}" {"a":1,"b":{"c":[1,2]},"s":"x"} +4 1 "42" 42 +5 0 "abc" abc + +-- !json_parse -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} +4 42 + +-- !typed_scalars -- +string "2026-01-02 03:04:05.000000" Decimal32 ["2026-01-02",1.50] + diff --git a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out index ff0c49b7bc445c..ceb60826c894fb 100644 --- a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out +++ b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out @@ -1,4 +1,106 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_order -- +2 +3 +1 + +-- !variant_topn -- +1 +3 + +-- !variant_window -- +1 3 +2 1 +3 2 + +-- !asc_nulls_last -- +10 +2 +3 +1 +4 +6 +7 +8 +9 +5 + +-- !desc_nulls_first -- +5 +9 +7 +8 +6 +4 +1 +2 +3 +10 + +-- !rank_peers -- +1 5 4 +2 3 3 +3 3 3 +4 6 5 +5 1 1 +6 7 6 +7 8 7 +8 8 7 +9 10 8 +10 2 2 + +-- !partition_topn -- +5 1 +10 2 +2 3 +3 4 + +-- !spill_sort -- +5 +10 +2 +3 +1 +4 +6 +7 +8 +9 + +-- !spill_sort_desc -- +9 +7 +8 +6 +4 +1 +2 +3 +10 +5 + +-- !bare_null_equals -- +2 \N \N \N \N +5 \N \N \N \N +6 \N \N \N \N + +-- !bare_null_safe_equals -- +5 + +-- !bare_null_safe_equals_reversed -- +5 + +-- !bare_null_safe_not_equals -- +1 +10 +2 +3 +4 +6 +7 +8 +9 + -- !explicit_cast_order -- 1 2 diff --git a/regression-test/data/variant_p0/test_variant_relational_corners.out b/regression-test/data/variant_p0/test_variant_relational_corners.out new file mode 100644 index 00000000000000..54b9646c89aded --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_relational_corners.out @@ -0,0 +1,342 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !constant_types -- +false false false false false + +-- !decimal_types -- +true false true true + +-- !extracted -- +1 1 false +2 1 false +3 1 false +5 0 false +6 -0 false +8 \N true +9 \N true +10 \N true +11 [1,2] false +12 [1,2] false +13 [2,1] false +14 {"a":1,"b":2} false +15 {"a":1,"b":2} false +16 [] false +17 \N true +18 9007199254740992 false +19 9007199254740993 false +20 9007199254740992 false +21 1.5 false +22 false + +-- !groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !distinct -- +11 + +-- !native_order -- +8 +9 +10 +17 +5 +6 +1 +2 +21 +18 +20 +19 +22 +3 +14 +15 +16 +11 +12 +13 + +-- !native_order_desc -- +13 +11 +12 +16 +14 +15 +3 +22 +19 +18 + +-- !roundtrip_types -- +1 false true false +2 false true false +3 false false false +8 \N \N \N +9 \N \N \N +17 \N \N \N + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !cast_precision -- +18 18 +18 19 +18 20 +19 18 +19 19 +19 20 +20 18 +20 19 +20 20 + +-- !cast_groups -- +1 3 + +-- !cast_order -- +5 0 +6 -0 +1 1 +2 1 +21 1.5 +18 9007199254740992 +19 9007199254740992 +20 9007199254740992 + +-- !numeric_promotion -- +1 9007199254740992 +2 9007199254740993 +3 9007199254740992 +4 1 +5 true +6 false +7 0 + +-- !numeric_groups -- +1 2 +2 1 +4 1 +5 1 +6 1 +7 1 + +-- !numeric_join -- +1 1 +1 3 +2 2 +3 1 +3 3 +4 4 +5 5 +6 6 +7 7 + +-- !spill_groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !spill_join -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + diff --git a/regression-test/data/variant_p0/test_variant_subfield_comparison.out b/regression-test/data/variant_p0/test_variant_subfield_comparison.out new file mode 100644 index 00000000000000..681672733562bc --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_subfield_comparison.out @@ -0,0 +1,610 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 \N \N \N \N true \N \N true true true +21 \N 1 \N \N false \N \N false true false +22 1 \N \N \N false \N \N false false true +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 \N \N \N \N true \N \N true true true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 \N \N \N \N true \N \N true true true +37 \N [] \N \N false \N \N false true false +38 \N \N \N \N true \N \N true true true +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +28 +29 +31 +33 +35 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +26 +27 +28 +29 +31 +33 +35 +36 +38 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +21 +24 +25 +26 +27 +28 +29 +31 +33 +35 +36 +37 +38 + +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 null null true false true true false true false false +21 null 1 false true false false true false false false +22 1 null false true false false true false false false +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 null \N \N \N false \N \N false false true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 {} {} true false true true false true false false +37 {} [] false true false false true false false false +38 {"x":null} {} false true false false true false false false +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +24 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !null_object_roundtrip -- +20 true false true false true true +21 true false false false false false +26 true false true true true false +36 true false true false true true +37 true false false false false false +38 true false true false true false + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false false true false false true false +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- + +-- !sql -- +1 +2 + +-- !sql -- +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !explicit_numeric_cast -- +1 true false false +2 false false true +3 true false false +4 false false true +5 true false false +6 false false true +7 true false false +8 true false false +9 true false false +10 true false false +11 true false false +12 true false false + +-- !typed_projection -- +1 1 1 true false true +2 1 1.5 false true false +3 0 -0 true false true +4 9007199254740993 9007199254740992 false true false +5 9007199254740992 9007199254740992 true false true +6 \N 1 \N \N false +7 1 \N \N \N false +8 \N \N \N \N true +9 \N \N \N \N true + +-- !sql -- +1 +3 +5 + +-- !sql -- +2 +4 + +-- !sql -- +1 +3 +5 +8 +9 + diff --git a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out b/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out deleted file mode 100644 index 05485b53ca8152..00000000000000 --- a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_all -- -1 2003-12-21 10:29:00.000000+08:00 2003-12-21 10:29:00.000000+08:00 2003-12-21 10:29:00.000000+08:00 2003-12-21 15:29:00.000000+08:00 2003-12-21 15:29:00.000000+08:00 2003-12-21 07:29:00.000000+08:00 - --- !select_cast_ts0 -- -1 2003-12-21 10:29:00+08:00 2003-12-21 10:29:00.000+08:00 2003-12-21 15:29:00.000000+08:00 - diff --git a/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out b/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out new file mode 100644 index 00000000000000..2d7ef477cf5f15 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !scalar_comparison -- +true true true + +-- !json_extract -- +1 + diff --git a/regression-test/data/variant_p0/test_variant_window_partition.out b/regression-test/data/variant_p0/test_variant_window_partition.out new file mode 100644 index 00000000000000..c2c30815eb02b1 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_window_partition.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_by_subpath -- +1 1 1 +2 1 2 +3 2 1 +4 a 1 +5 true 1 +6 true 2 +7 1 3 +8 true 3 + +-- !partition_by_whole -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_rank -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_topn -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 + +-- !partition_mixed_keys -- +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 2 +8 2 + +-- !many_blocks_typed_keys -- +5000 12502500 4169167 + +-- !many_blocks_encoded_keys -- +5000 12502500 4169167 + diff --git a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out index 80fea8c68d269b..9e635b956753a9 100644 --- a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out +++ b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out @@ -1,4 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !equals -- +true + +-- !not_equals -- +false + +-- !null_safe_equals -- +true + +-- !equality_join -- +0 0 +1 1 + +-- !null_safe_equality_join -- +0 0 +1 1 + -- !group_by -- 0 2 1 2 @@ -36,3 +53,4 @@ false null 1 -- !explicit_cast_equality -- true true + diff --git a/regression-test/suites/nereids_p0/join/test_join_on.groovy b/regression-test/suites/nereids_p0/join/test_join_on.groovy index 7de75912796cc6..c0b742514c544a 100644 --- a/regression-test/suites/nereids_p0/join/test_join_on.groovy +++ b/regression-test/suites/nereids_p0/join/test_join_on.groovy @@ -47,8 +47,12 @@ suite("test_join_on", "nereids_p0") { exception "errCode = 2" } - test { - sql """select * from (select cast('' as variant) as a) t1 join (select cast('' as variant) as a) t2 on t1.a = t2.a""" - exception "could not used in ComparisonPredicate (a = a)" + setFeConfigTemporary([enable_variant_v2: true]) { + order_qt_variant_join """ + select t1.id, t2.id + from (select 1 as id, cast('x' as variant) as a union all select 2, cast(1 as variant)) t1 + join (select 3 as id, parse_to_variant('"x"') as a union all select 4, parse_to_variant('1.0')) t2 + on t1.a = t2.a + """ } } diff --git a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy index c2c6c236f1d68b..8e7023ccfafb8c 100644 --- a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy +++ b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy @@ -41,7 +41,6 @@ suite("test_predefine_typed_to_sparse_1shard", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_common_expr_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - // sql """ set enable_variant_schema_auto_cast = false """ def load_json_data = {table_name, file_name -> // load the json data diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy index f832ccc716d1f1..89da23cdf36951 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy @@ -107,6 +107,12 @@ suite("test_predefine_ddl", "p0") { sql """ alter table ${tableName} add column var3 variant<'ab' : string> NULL """ + // A Schema Template cannot declare TIMESTAMPTZ paths. + test { + sql """ alter table ${tableName} add column var4 variant<'ts' : timestamptz(3)> NULL """ + exception("VARIANT unsupported sub-type: timestamptz(3)") + } + // TODO(lihangyu) : uncomment // test { // sql """ alter table ${tableName} modify column var variant<'ab' : string> NULL """ @@ -202,6 +208,30 @@ suite("test_predefine_ddl", "p0") { exception("VARIANT unsupported sub-type: datetime") } + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : timestamptz(6) + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: timestamptz(6)") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + 'ab*' : array + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: array r.v""" + order_qt_composite_null_safe """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v <=> r.v AND l.id % 2 = r.id % 2""" + order_qt_left """SELECT l.id, r.id FROM variant_join_left l + LEFT JOIN variant_join_right r ON l.v = r.v""" + order_qt_full """SELECT l.id, r.id FROM variant_join_left l + FULL JOIN variant_join_right r ON l.v = r.v""" + order_qt_semi """SELECT l.id FROM variant_join_left l + LEFT SEMI JOIN variant_join_right r ON l.v = r.v""" + order_qt_anti """SELECT l.id FROM variant_join_left l + LEFT ANTI JOIN variant_join_right r ON l.v = r.v""" + order_qt_in """SELECT id FROM variant_join_left + WHERE v IN (SELECT v FROM variant_join_right)""" + order_qt_not_in """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right)""" + order_qt_mark """SELECT l.id, l.v IN (SELECT r.v FROM variant_join_right r) + FROM variant_join_left l""" + order_qt_exists """SELECT l.id, EXISTS(SELECT 1 FROM variant_join_right r WHERE l.v = r.v) + FROM variant_join_left l""" + order_qt_subpath """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v['id'] = r.v['id']""" + order_qt_residual """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v = r.v OR l.id = r.id""" + order_qt_scalar_equality """SELECT l.id, l.v = parse_to_variant('1.0'), + l.v != parse_to_variant('1.0'), l.v <=> CAST(NULL AS VARIANT) + FROM variant_join_left l""" + qt_json_null_equality """SELECT parse_to_variant('null') = parse_to_variant('null'), + parse_to_variant('null') = parse_to_variant('{}')""" + // Without a NULL on the build side, NOT IN depends on canonical key matching. + order_qt_not_in_non_null """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right WHERE v IS NOT NULL)""" + // A typed key from CAST(scalar AS VARIANT) against a stored Variant key. + order_qt_typed_key """SELECT l.id, r.id + FROM (SELECT id, CAST(CAST(id AS BIGINT) AS VARIANT) k FROM variant_join_left) l + JOIN variant_join_right r ON l.k = r.v""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + order_qt_spill """SELECT l.id, r.id FROM variant_join_left l + JOIN [shuffle] variant_join_right r ON l.v <=> r.v""" + test { + sql "SELECT * FROM variant_join_left l JOIN variant_join_right r ON l.v > r.v" + exception "CAST to a concrete type first" + } + } +} diff --git a/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy new file mode 100644 index 00000000000000..d6088108bccb93 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_join_type_properties", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + // Variant join keys compare canonically. Keys whose VARIANT types differ only in + // properties (max subcolumns count, doc mode) must join like identical types. + sql "DROP TABLE IF EXISTS variant_join_props_a" + sql "DROP TABLE IF EXISTS variant_join_props_b" + sql "DROP TABLE IF EXISTS variant_join_props_c" + sql "DROP TABLE IF EXISTS variant_join_props_d" + sql """CREATE TABLE variant_join_props_a ( + id INT, v VARIANT, s STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_b ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_c ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_d ( + id INT, + v VARIANT, + w VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_join_props_a VALUES + (1, parse_to_variant('1'), '1.0'), (2, parse_to_variant('"x"'), '"x"'), + (3, NULL, NULL), (4, parse_to_variant('{"k":1,"j":[1,2]}'), '{"j":[1.0,2],"k":1.0}'), + (5, parse_to_variant('true'), '1')""" + sql """INSERT INTO variant_join_props_b VALUES + (1, parse_to_variant('1.0')), (2, parse_to_variant('"x"')), (3, NULL), + (4, parse_to_variant('{"j":[1,2.0],"k":1}')), (6, parse_to_variant('false'))""" + sql """INSERT INTO variant_join_props_c VALUES + (1, parse_to_variant('1')), (4, parse_to_variant('{"k":1.0,"j":[1,2]}'))""" + sql """INSERT INTO variant_join_props_d VALUES + (1, parse_to_variant('1'), parse_to_variant('1.0')), + (2, parse_to_variant('"x"'), parse_to_variant('"x"')), + (4, parse_to_variant('{"k":1}'), parse_to_variant('{"k":2}'))""" + + // Different max subcolumns counts. + order_qt_broadcast """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [broadcast] variant_join_props_b b ON a.v = b.v""" + order_qt_shuffle """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [shuffle] variant_join_props_b b ON a.v = b.v""" + order_qt_null_safe """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v <=> b.v""" + order_qt_subpath """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v['j'] = b.v['j']""" + order_qt_left_semi """SELECT a.id FROM variant_join_props_a a + LEFT SEMI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_left_anti """SELECT a.id FROM variant_join_props_a a + LEFT ANTI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_mark """SELECT a.id, a.v IN (SELECT b.v FROM variant_join_props_b b) + FROM variant_join_props_a a""" + order_qt_not_in """SELECT a.id FROM variant_join_props_a a + WHERE a.v NOT IN (SELECT b.v FROM variant_join_props_b b WHERE b.v IS NOT NULL)""" + // Different doc mode. + order_qt_doc_mode """SELECT a.id, c.id FROM variant_join_props_a a + JOIN variant_join_props_c c ON a.v = c.v""" + // A stored column against a computed Variant whose type has default properties. + order_qt_computed_key """SELECT l.id, r.id FROM variant_join_props_a l + JOIN variant_join_props_a r ON l.v = parse_to_variant(r.s)""" + // The optimizer infers a.v = b.v from d.v = a.v, d.w = b.v and d.v = d.w and uses it + // as a hash key between tables whose Variant properties differ. + sql "SET disable_join_reorder = true" + order_qt_inferred_key """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.id = b.id + JOIN variant_join_props_d d ON d.v = a.v AND d.w = b.v + WHERE d.v = d.w""" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_json_functions.groovy b/regression-test/suites/variant_p0/test_variant_json_functions.groovy new file mode 100644 index 00000000000000..eaf546afe18583 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// JSON functions accept a Variant argument by casting it to JSON implicitly. +suite("test_variant_json_functions", "p0") { + setFeConfigTemporary([enable_variant_v2: true]) { + + sql "DROP TABLE IF EXISTS test_variant_json_functions" + sql """ + CREATE TABLE test_variant_json_functions (k INT, v VARIANT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO test_variant_json_functions + SELECT k, parse_to_variant(doc) FROM ( + SELECT 1 AS k, '{"a":1,"s":"x","b":{"c":[1,2]}}' AS doc + UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}' + UNION ALL SELECT 3, '[1,"two",{"three":3}]' + UNION ALL SELECT 4, '42' + UNION ALL SELECT 5, 'abc' + UNION ALL SELECT 6, NULL) t + """ + + // json_insert and json_set reject a scalar document, so they only run on object rows. + order_qt_insert_set_replace_remove """ + SELECT k, json_insert(v, '\$.z', 9), json_set(v, '\$.a', 0), json_replace(v, '\$.s', 'r'), + json_remove(v, '\$.a') + FROM test_variant_json_functions WHERE k IN (1, 2, 6) ORDER BY k + """ + order_qt_extract """ + SELECT k, json_extract(v, '\$.b.c'), json_extract_string(v, '\$.s'), json_extract_int(v, '\$.a'), + json_extract_bigint(v, '\$.a'), json_extract_double(v, '\$.a'), json_extract_bool(v, '\$.n.m'), + json_extract_isnull(v, '\$.a'), jsonb_extract(v, '\$[1]'), get_json_string(v, '\$.s'), + get_json_int(v, '\$.a') + FROM test_variant_json_functions ORDER BY k + """ + order_qt_inspect """ + SELECT k, json_keys(v), json_length(v), json_type(v, '\$'), json_exists_path(v, '\$.s'), + json_contains(v, '1', '\$.a'), json_search(v, 'one', 'x'), json_hash(v), + json_object_flatten(v), sort_json_object_keys(v), normalize_json_numbers_to_double(v) + FROM test_variant_json_functions ORDER BY k + """ + // Variant values nested into new JSON documents keep their JSON structure. + order_qt_construct """ + SELECT k, to_json(v), json_array(v, 1), json_array_ignore_null(v, NULL), json_object('v', v), + json_insert('{}', '\$.v', v), json_set('{}', '\$.b', v['b']), json_replace('{"v":0}', '\$.v', v['a']) + FROM test_variant_json_functions ORDER BY k + """ + + // Variant elements of an ARRAY convert the same way. + order_qt_array_of_variant """ + SELECT k, to_json(CAST(v AS ARRAY)), CAST(CAST(v AS ARRAY) AS JSON), + json_array(CAST(v AS ARRAY)) + FROM test_variant_json_functions WHERE k = 3 + """ + + // The implicit cast gives the same result as the explicit one. + order_qt_same_as_explicit_cast """ + SELECT k, + CAST(json_replace(v, '\$.s', 'r') AS STRING) <=> CAST(json_replace(CAST(v AS JSON), '\$.s', 'r') AS STRING), + CAST(json_extract(v, '\$.b') AS STRING) <=> CAST(json_extract(CAST(v AS JSON), '\$.b') AS STRING), + CAST(json_keys(v) AS STRING) <=> CAST(json_keys(CAST(v AS JSON)) AS STRING), + json_hash(v) <=> json_hash(CAST(v AS JSON)), + CAST(to_json(v) AS STRING) <=> CAST(CAST(v AS JSON) AS STRING), + CAST(json_array(v) AS STRING) <=> CAST(json_array(CAST(v AS JSON)) AS STRING) + FROM test_variant_json_functions ORDER BY k + """ + + // Functions with a string signature keep treating Variant as text, so an unparsable document + // that was stored as a Variant string is still reported as invalid JSON. + order_qt_string_signature """ + SELECT k, json_valid(v), json_quote(v), json_unquote(v) + FROM test_variant_json_functions WHERE k IN (1, 4, 5) ORDER BY k + """ + order_qt_json_parse """ + SELECT k, json_parse(v) FROM test_variant_json_functions WHERE k IN (1, 4) ORDER BY k + """ + + // Typed Variant scalars without a JSON counterpart become JSON strings. + order_qt_typed_scalars """ + SELECT json_type(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), '\$'), + json_extract(CAST(CAST('2026-01-02 03:04:05' AS DATETIME) AS VARIANT), '\$'), + json_type(CAST(CAST(1.50 AS DECIMAL(5, 2)) AS VARIANT), '\$'), + json_array(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), CAST(1.50 AS DECIMAL(5, 2))) + """ + } +} diff --git a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy index ee0b25cb62f9db..1e329c1c10588c 100644 --- a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy +++ b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy @@ -16,54 +16,91 @@ // under the License. suite("test_variant_ordering_comparison_error", "p0,nonConcurrent") { - def enableVariantV2 = getFeConfig("enable_variant_v2").toBoolean() + setFeConfigTemporary([enable_variant_v2: true]) { + sql "SET enable_nereids_planner = true" sql "SET enable_fallback_to_original_planner = false" - test { - sql """ - SELECT v + qt_variant_order """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v + ORDER BY v, id """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" - } - - test { - sql """ - SELECT v + qt_variant_topn """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v - LIMIT 1 + ORDER BY v DESC, id + LIMIT 2 """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" + qt_variant_window """ + SELECT id, row_number() OVER (ORDER BY v, id) + FROM ( + SELECT 1 AS id, parse_to_variant('2') AS v + UNION ALL + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v + ) t + ORDER BY id + """ + // Canonical peers (1, 1.0 and objects with reordered keys), SQL NULL, a JSON null (read back + // as {}) and a boolean. One bucket keeps the numbers ahead of the boolean in the segment. + sql "DROP TABLE IF EXISTS variant_ordering_peers" + sql """CREATE TABLE variant_ordering_peers (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_ordering_peers VALUES + (1, parse_to_variant('2')), (2, parse_to_variant('1')), (3, parse_to_variant('1.0')), + (4, parse_to_variant('"a"')), (5, NULL), (6, parse_to_variant('null')), + (7, parse_to_variant('{"k":1,"j":2}')), (8, parse_to_variant('{"j":2,"k":1.0}')), + (9, parse_to_variant('[1,2]')), (10, parse_to_variant('true'))""" + qt_asc_nulls_last "SELECT id FROM variant_ordering_peers ORDER BY v ASC NULLS LAST, id" + qt_desc_nulls_first "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS FIRST, id" + qt_rank_peers """SELECT id, rank() OVER (ORDER BY v), dense_rank() OVER (ORDER BY v) + FROM variant_ordering_peers ORDER BY id""" + qt_partition_topn """SELECT id, rn FROM ( + SELECT id, row_number() OVER (ORDER BY v, id) rn FROM variant_ordering_peers) t + WHERE rn <= 4 ORDER BY rn""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_sort "SELECT id FROM variant_ordering_peers ORDER BY v NULLS FIRST, id" + qt_spill_sort_desc "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS LAST, id" + sql "SET enable_force_spill = false" + sql "SET enable_spill = false" + + // A bare NULL takes the Variant type on either side. `=` and `!=` are NULL for every row; + // `<=>` matches the SQL NULL row only. + order_qt_bare_null_equals """SELECT id, v = NULL, NULL = v, v != NULL, NULL != v + FROM variant_ordering_peers WHERE id IN (2, 5, 6)""" + order_qt_bare_null_safe_equals "SELECT id FROM variant_ordering_peers WHERE v <=> NULL" + order_qt_bare_null_safe_equals_reversed "SELECT id FROM variant_ordering_peers WHERE NULL <=> v" + order_qt_bare_null_safe_not_equals "SELECT id FROM variant_ordering_peers WHERE NOT (v <=> NULL)" + test { + sql "SELECT id FROM variant_ordering_peers WHERE v > NULL" + exception "CAST to a concrete type first" } test { sql "SELECT CAST('2' AS VARIANT) > CAST('1' AS VARIANT)" - exception enableVariantV2 - ? "CAST to a concrete type first" - : "could not used in ComparisonPredicate" + exception "CAST to a concrete type first" } - if (enableVariantV2) { - test { - sql "SELECT CAST('2' AS VARIANT) > 1" - exception "CAST to a concrete type first" - } - - test { - sql "SELECT CAST('2' AS VARIANT) <=> CAST('1' AS VARIANT)" - exception "CAST to a concrete type first" - } + test { + sql "SELECT CAST('2' AS VARIANT) > 1" + exception "CAST to a concrete type first" } order_qt_explicit_cast_order """ @@ -111,4 +148,5 @@ suite("test_variant_ordering_comparison_error", "p0,nonConcurrent") { exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" } + } } diff --git a/regression-test/suites/variant_p0/test_variant_relational_corners.groovy b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy new file mode 100644 index 00000000000000..c255471a2282fa --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_relational_corners", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + qt_constant_types """SELECT + parse_to_variant('true') = parse_to_variant('1'), + parse_to_variant('false') = parse_to_variant('0'), + parse_to_variant('null') = parse_to_variant('{}'), + parse_to_variant('{"k":{}}')['k'] IS NULL, + parse_to_variant('9007199254740993') = parse_to_variant('9007199254740992.0')""" + qt_decimal_types """SELECT + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) + = CAST(CAST(1.500 AS DECIMAL(10,3)) AS VARIANT), + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1.5'), + CAST(CAST(1.00 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1'), + CAST(CAST(-1 AS TINYINT) AS VARIANT) = CAST(CAST(-1 AS BIGINT) AS VARIANT)""" + sql "DROP TABLE IF EXISTS variant_relational_corners" + sql """CREATE TABLE variant_relational_corners (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + // Keep values inside objects so extraction exercises shredded subcolumns, + // mixed-type promotion and the missing-path validity map. Booleans are compared in + // variant_relational_numeric instead: in this bucket layout a segment can see a boolean + // before a number, and storage then widens the boolean to an integer. + sql """INSERT INTO variant_relational_corners VALUES + (1, parse_to_variant('{"k":1}')), + (2, parse_to_variant('{"k":1.0}')), + (3, parse_to_variant('{"k":"1"}')), + (5, parse_to_variant('{"k":0}')), + (6, parse_to_variant('{"k":-0.0}')), + (8, parse_to_variant('{"k":null}')), + (9, parse_to_variant('{}')), + (10, NULL), + (11, parse_to_variant('{"k":[1,2]}')), + (12, parse_to_variant('{"k":[1.0,2.0]}')), + (13, parse_to_variant('{"k":[2,1]}')), + (14, parse_to_variant('{"k":{"a":1,"b":2}}')), + (15, parse_to_variant('{"k":{"b":2.0,"a":1.0}}')), + (16, parse_to_variant('{"k":[]}')), + (17, parse_to_variant('{"k":{}}')), + (18, parse_to_variant('{"k":9007199254740992}')), + (19, parse_to_variant('{"k":9007199254740993}')), + (20, parse_to_variant('{"k":9007199254740992.0}')), + (21, parse_to_variant('{"k":1.5}')), + (22, parse_to_variant('{"k":""}'))""" + + qt_extracted """SELECT id, v['k'], v['k'] IS NULL + FROM variant_relational_corners ORDER BY id""" + qt_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_distinct "SELECT count(DISTINCT v['k']) FROM variant_relational_corners" + qt_native_order """SELECT id FROM variant_relational_corners + ORDER BY v['k'] NULLS FIRST, id""" + qt_native_order_desc """SELECT id FROM variant_relational_corners + ORDER BY v['k'] DESC NULLS LAST, id LIMIT 10""" + // Compare stored values to constants explicitly: generated output is an + // observation of round-trip behavior, not a definition of type coercion. + qt_roundtrip_types """SELECT id, v['k'] = parse_to_variant('true'), + v['k'] = parse_to_variant('1'), v['k'] = parse_to_variant('{}') + FROM variant_relational_corners WHERE id IN (1,2,3,8,9,17) + ORDER BY id""" + ["broadcast", "shuffle"].each { distribution -> + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] = r.v['k'] + ORDER BY l.id, r.id""" + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" + } + // CAST is intentionally a different oracle here: strings can merge with + // numbers and DOUBLE cannot distinguish adjacent integers above 2^53. + qt_cast_precision """SELECT l.id, r.id FROM variant_relational_corners l + JOIN variant_relational_corners r + ON CAST(l.v['k'] AS DOUBLE) = CAST(r.v['k'] AS DOUBLE) + WHERE l.id BETWEEN 18 AND 20 AND r.id BETWEEN 18 AND 20 + ORDER BY l.id, r.id""" + qt_cast_groups """SELECT min(id), count(*) FROM variant_relational_corners + WHERE id IN (1,2,3) GROUP BY CAST(v['k'] AS BIGINT) ORDER BY min(id)""" + qt_cast_order """SELECT id, CAST(v['k'] AS DOUBLE) + FROM variant_relational_corners WHERE id IN (1,2,5,6,18,19,20,21) + ORDER BY CAST(v['k'] AS DOUBLE), id""" + sql "DROP TABLE IF EXISTS variant_relational_numeric" + sql """CREATE TABLE variant_relational_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_numeric VALUES + (1, parse_to_variant('{"k":9007199254740992}')), + (2, parse_to_variant('{"k":9007199254740993}')), + (3, parse_to_variant('{"k":9007199254740992.0}')), + (4, parse_to_variant('{"k":1}')), + (5, parse_to_variant('{"k":true}')), + (6, parse_to_variant('{"k":false}')), + (7, parse_to_variant('{"k":0}'))""" + qt_numeric_promotion """SELECT id, v['k'] FROM variant_relational_numeric ORDER BY id""" + qt_numeric_groups """SELECT min(id), count(*) FROM variant_relational_numeric + GROUP BY v['k'] ORDER BY min(id)""" + qt_numeric_join """SELECT l.id, r.id FROM variant_relational_numeric l + JOIN variant_relational_numeric r ON l.v['k'] = r.v['k'] ORDER BY l.id,r.id""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_spill_join """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [shuffle] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy new file mode 100644 index 00000000000000..aa8de479fcaa65 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_subfield_comparison", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + // Keep the JSON text as well as its stored Variant representation: reading a + // subcolumn can promote types, while parsing the text preserves each input. + sql "DROP TABLE IF EXISTS variant_subfield_comparison" + sql """CREATE TABLE variant_subfield_comparison (id INT, doc STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_comparison (id, doc) VALUES + (1, '{"a":1,"b":1}'), + (2, '{"a":1,"b":2}'), + (3, '{"a":-1,"b":-1}'), + (4, '{"a":127,"b":128}'), + (5, '{"a":1,"b":1.0}'), + (6, '{"a":1,"b":1.5}'), + (7, '{"a":1.5,"b":1.5}'), + (8, '{"a":0,"b":-0.0}'), + (9, '{"a":9007199254740993,"b":9007199254740992.0}'), + (10, '{"a":9007199254740992,"b":9007199254740992.0}'), + (11, '{"a":"1","b":1}'), + (12, '{"a":"1.5","b":1.5}'), + (13, '{"a":"abc","b":"abc"}'), + (14, '{"a":"abc","b":"abd"}'), + (15, '{"a":"","b":""}'), + (16, '{"a":true,"b":true}'), + (17, '{"a":true,"b":false}'), + (18, '{"a":true,"b":1}'), + (19, '{"a":false,"b":0}'), + (20, '{"a":null,"b":null}'), + (21, '{"a":null,"b":1}'), + (22, '{"a":1,"b":null}'), + (23, '{"a":1}'), + (24, '{"b":1}'), + (25, '{}'), + (26, '{"a":null}'), + (27, NULL), + (28, '{"a":[1,2],"b":[1,2]}'), + (29, '{"a":[1,2],"b":[1.0,2.0]}'), + (30, '{"a":[1,2],"b":[2,1]}'), + (31, '{"a":[],"b":[]}'), + (32, '{"a":[1],"b":1}'), + (33, '{"a":[1,null,"x",[2]],"b":[1.0,null,"x",[2.0]]}'), + (34, '{"a":[null],"b":[]}'), + (35, '{"a":{"x":1,"y":2},"b":{"y":2.0,"x":1.0}}'), + (36, '{"a":{},"b":{}}'), + (37, '{"a":{},"b":[]}'), + (38, '{"a":{"x":null},"b":{}}'), + (39, '{"a":[1],"b":["1"]}'), + (40, '{"a":[{"x":1}],"b":[{"x":2}]}')""" + // Insert into a second table to retain both original text and parsed values. + sql "DROP TABLE IF EXISTS variant_subfield_values" + sql """CREATE TABLE variant_subfield_values (id INT, doc STRING, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_values + SELECT id, doc, parse_to_variant(doc) FROM variant_subfield_comparison""" + + // These are separate round-trip expectations, not an assertion that storage + // preserves JSON null and empty objects. Rows 20-27 and 36-38 expose the + // current difference between stored-path SQL NULL and parsed JSON values. + ["v", "parse_to_variant(doc)"].each { expression -> + qt_sql """SELECT id, x['a'], x['b'], + x['a'] = x['b'], x['a'] != x['b'], x['a'] <=> x['b'], + x['b'] = x['a'], x['b'] != x['a'], x['b'] <=> x['a'], + x['a'] IS NULL, x['b'] IS NULL + FROM (SELECT id, ${expression} x FROM variant_subfield_values) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b'] ORDER BY id""" + } + qt_sql """SELECT id FROM variant_subfield_values + WHERE NOT (${expression}['a'] = ${expression}['b']) ORDER BY id""" + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] = ${expression}['b'] + OR ${expression}['a'] IS NULL ORDER BY id""" + } + // TopN ordering is supported; relational ordering predicates still require CAST. + ["v", "parse_to_variant(doc)"].each { expression -> + [">", ">=", "<", "<="].each { op -> + test { + sql "SELECT ${expression}['a'] ${op} ${expression}['b'] FROM variant_subfield_values" + exception "CAST to a concrete type first" + } + test { + sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b']""" + exception "CAST to a concrete type first" + } + } + } + // Isolate the known storage round-trip differences from ordinary equality cases. + qt_null_object_roundtrip """SELECT id, + v['a'] IS NULL, parse_to_variant(doc)['a'] IS NULL, + v['b'] IS NULL, parse_to_variant(doc)['b'] IS NULL, + v['a'] <=> v['b'], + parse_to_variant(doc)['a'] <=> parse_to_variant(doc)['b'] + FROM variant_subfield_values WHERE id IN (20,21,26,36,37,38) ORDER BY id""" + + // JSON text cannot retain Decimal, date or IP physical types. Cast scalar + // columns to Variant directly to exercise these same-row comparison types. + sql "DROP TABLE IF EXISTS variant_comparison_scalar_sources" + sql """CREATE TABLE variant_comparison_scalar_sources + (id INT, a STRING, b STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + [ + ["DECIMAL(18,2)", "DECIMAL(38,3)", "1.50", "1.500", "1.501"], + ["DECIMAL(18,2)", "DOUBLE", "1.50", "1.5", "1.6"], + ["DECIMAL(18,2)", "BIGINT", "1.00", "1", "2"], + ["DATEV2", "DATEV2", "2024-02-29", "2024-02-29", "2024-03-01"], + ["DATETIMEV2(3)", "DATETIMEV2(6)", "2024-02-29 12:34:56.123", + "2024-02-29 12:34:56.123000", "2024-02-29 12:34:56.123001"], + ["IPV4", "IPV4", "10.0.0.2", "10.0.0.2", "10.0.0.10"], + ["IPV6", "IPV6", "2001:db8::1", "2001:0db8:0:0:0:0:0:1", "2001:db8::2"], + ["DOUBLE", "DOUBLE", "NaN", "NaN", "Infinity"], + ["DOUBLE", "DOUBLE", "Infinity", "Infinity", "-Infinity"] + ].each { spec -> + sql "TRUNCATE TABLE variant_comparison_scalar_sources" + sql """INSERT INTO variant_comparison_scalar_sources VALUES + (1, '${spec[2]}', '${spec[3]}'), (2, '${spec[2]}', '${spec[4]}'), + (3, NULL, '${spec[3]}'), (4, '${spec[2]}', NULL), (5, NULL, NULL)""" + def source = """SELECT id, CAST(CAST(a AS ${spec[0]}) AS VARIANT) a, + CAST(CAST(b AS ${spec[1]}) AS VARIANT) b + FROM variant_comparison_scalar_sources""" + qt_sql """SELECT id, a IS NULL, b IS NULL, + a = b, a != b, a <=> b, b = a, b != a, b <=> a + FROM (${source}) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql "SELECT id FROM (${source}) t WHERE a ${op} b ORDER BY id" + } + // Typed Variant values also order natively; cover both NULL placements. + qt_sql "SELECT id FROM (${source}) t ORDER BY b ASC NULLS LAST, id" + qt_sql "SELECT id FROM (${source}) t ORDER BY b DESC NULLS FIRST, id" + } + qt_explicit_numeric_cast """SELECT id, + CAST(v['a'] AS DOUBLE) = CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) > CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) < CAST(v['b'] AS DOUBLE) + FROM variant_subfield_values WHERE id BETWEEN 1 AND 12 ORDER BY id""" + + // Homogeneous paths exercise typed subcolumns with different numeric widths. + sql "DROP TABLE IF EXISTS variant_subfield_numeric" + sql """CREATE TABLE variant_subfield_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_numeric VALUES + (1, parse_to_variant('{"a":1,"b":1.0}')), + (2, parse_to_variant('{"a":1,"b":1.5}')), + (3, parse_to_variant('{"a":0,"b":-0.0}')), + (4, parse_to_variant('{"a":9007199254740993,"b":9007199254740992.0}')), + (5, parse_to_variant('{"a":9007199254740992,"b":9007199254740992.0}')), + (6, parse_to_variant('{"a":null,"b":1.0}')), + (7, parse_to_variant('{"a":1}')), + (8, parse_to_variant('{}')), + (9, NULL)""" + qt_typed_projection """SELECT id, v['a'], v['b'], + v['a'] = v['b'], v['a'] != v['b'], v['a'] <=> v['b'] + FROM variant_subfield_numeric ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_numeric + WHERE v['a'] ${op} v['b'] ORDER BY id""" + } + } +} diff --git a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy b/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy deleted file mode 100644 index 057166560d3531..00000000000000 --- a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Reproduces DORIS-25915: -// When VARIANT has typed paths that exceed variant_max_subcolumns_count and -// variant_enable_typed_paths_to_sparse=true, timestamptz values that fall to -// the sparse path are read back as only the timezone suffix (e.g. "+00:00") -// instead of the full timestamp. -suite("test_variant_timestamptz_sparse", "p0"){ - def variantV2Function = getFeConfig("enable_variant_v2").toBoolean() ? "parse_to_variant" : "" - sql " set time_zone = '+08:00' " - sql " set default_variant_enable_doc_mode = false " - - sql "DROP TABLE IF EXISTS test_variant_timestamptz_sparse_repro" - - sql """ - CREATE TABLE test_variant_timestamptz_sparse_repro ( - pk int, - var VARIANT< - 'c1':bigint, - 'c2':bigint, - 'd1':date, - 'd2':date, - 'dt0':datetime(0), - 'dt0n':datetime(0), - 'dt3':datetime(3), - 'dt3n':datetime(3), - 'dt6':datetime(6), - 'dt6n':datetime(6), - 'ts0':timestamptz(0), - 'ts0n':timestamptz(0), - 'ts3':timestamptz(3), - 'ts3n':timestamptz(3), - 'ts6':timestamptz(6), - 'ts6n':timestamptz(6), - 'v1':string, - 'v2':string, - properties("variant_max_subcolumns_count" = "2","variant_enable_typed_paths_to_sparse" = "true") - > NULL, - INDEX c1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c1"), - INDEX c2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c2"), - INDEX d1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d1"), - INDEX d2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d2"), - INDEX dt0 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0"), - INDEX dt0n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0n"), - INDEX dt3 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3"), - INDEX dt3n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3n"), - INDEX dt6 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6"), - INDEX dt6n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6n"), - INDEX v1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v1"), - INDEX v2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v2") - ) ENGINE=OLAP - DUPLICATE KEY(pk) - DISTRIBUTED BY HASH(pk) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ - INSERT INTO test_variant_timestamptz_sparse_repro VALUES - (1, ${variantV2Function}('{"c1":1699516324,"c2":1690122421,"d1":"2024-02-25","d2":"2024-11-27","dt0":"2002-12-25 12:42:19","dt0n":"2015-02-05 02:36:13","dt3":"2015-02-22 02:09:01","dt3n":"2015-09-16 02:55:07","dt6":"2001-09-19 09:53:52","dt6n":"2003-12-21 02:29:00","ts0":"2003-12-21 02:29:00 +00:00","ts0n":"2003-12-21 02:29:00 +00:00","ts3":"2003-12-21 02:29:00 +00:00","ts3n":"2003-12-21 02:29:00 -05:00","ts6":"2003-12-21 02:29:00 -05:00","ts6n":"2003-12-21 02:29:00 +03:00","v1":"2024-12-10 10:16:19","v2":"2023-12-31 23:59:59"}')); - """ - - sql "SYNC" - - // Pre-fix: every column that fell to sparse returned just the timezone - // suffix ("+08:00"). Post-fix: each ts* path round-trips as a full timestamp. - qt_select_all """ - SELECT - pk, - CAST(CAST(var['ts0'] AS timestamptz(6)) AS string) AS str_ts0, - CAST(CAST(var['ts0n'] AS timestamptz(6)) AS string) AS str_ts0n, - CAST(CAST(var['ts3'] AS timestamptz(6)) AS string) AS str_ts3, - CAST(CAST(var['ts3n'] AS timestamptz(6)) AS string) AS str_ts3n, - CAST(CAST(var['ts6'] AS timestamptz(6)) AS string) AS str_ts6, - CAST(CAST(var['ts6n'] AS timestamptz(6)) AS string) AS str_ts6n - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ - - qt_select_cast_ts0 """ - SELECT - pk, - CAST(var['ts0'] AS timestamptz(0)) AS cast_ts0, - CAST(var['ts3'] AS timestamptz(3)) AS cast_ts3, - CAST(var['ts6'] AS timestamptz(6)) AS cast_ts6 - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ -} diff --git a/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy b/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy new file mode 100644 index 00000000000000..8f467380e5ed33 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_v1_relational_compatibility", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: false]) { + test { + sql ''' + SELECT v FROM ( + SELECT CAST('2' AS VARIANT) AS v + UNION ALL SELECT CAST('1' AS VARIANT) AS v + ) t ORDER BY v + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT v FROM ( + SELECT CAST('2' AS VARIANT) AS v + UNION ALL SELECT CAST('1' AS VARIANT) AS v + ) t ORDER BY v LIMIT 1 + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT row_number() OVER (ORDER BY parse_to_variant(CAST(number AS STRING))) + FROM numbers("number" = "2") + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT * FROM (SELECT CAST('1' AS VARIANT) AS v) a + JOIN (SELECT CAST('1' AS VARIANT) AS v) b ON a.v = b.v + ''' + exception "could not used in ComparisonPredicate" + } + order_qt_scalar_comparison ''' + SELECT CAST('1' AS VARIANT) = 1, CAST('2' AS VARIANT) > 1, + CAST(CAST('1' AS VARIANT) AS STRING) = '1' + ''' + order_qt_json_extract ''' + SELECT json_extract(CAST('{"id":1}' AS VARIANT), '$.id') + ''' + } +} diff --git a/regression-test/suites/variant_p0/test_variant_window_partition.groovy b/regression-test/suites/variant_p0/test_variant_window_partition.groovy new file mode 100644 index 00000000000000..ad479232c62485 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_window_partition.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// GROUP BY, ORDER BY, hash join and window ORDER BY accept Variant, and they all use the same +// canonical equality and comparison. Window PARTITION BY used to be the one remaining context that +// rejected it, so these queries pin that a Variant partition key groups rows by canonical value: +// values of different JSON kinds ("a", true, 1) never share a partition, and equal values do. +suite("test_variant_window_partition", "p0") { + setFeConfigTemporary([enable_variant_v2: true]) { + + sql "drop table if exists test_variant_window_partition" + + sql """ + create table test_variant_window_partition ( + id int, + v variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + + sql """ + insert into test_variant_window_partition values + (1, parse_to_variant('{"k": 1}')), + (2, parse_to_variant('{"k": 1}')), + (3, parse_to_variant('{"k": 2}')), + (4, parse_to_variant('{"k": "a"}')), + (5, parse_to_variant('{"k": true}')), + (6, parse_to_variant('{"k": true}')), + (7, parse_to_variant('{"k": 1}')), + (8, parse_to_variant('{"k": true}')) + """ + + // PARTITION BY a Variant sub-path. + order_qt_partition_by_subpath """ + select id, v['k'] as k, row_number() over (partition by v['k'] order by id) as rn + from test_variant_window_partition + """ + + // PARTITION BY the whole Variant value. + order_qt_partition_by_whole """ + select id, row_number() over (partition by v order by id) as rn + from test_variant_window_partition + """ + + // rank() with a Variant partition key. + order_qt_partition_rank """ + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + """ + + // CreatePartitionTopNFromWindow matches logicalFilter(logicalWindow()), so a PartitionTopN + // operator only appears when a filter sits above the window. That operator organises the + // Variant partition key as a serialized hash key on BE, which the queries above never reach. + order_qt_partition_topn """ + select id, rk from ( + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + ) t where rk <= 2 + """ + + // A Variant partition key combined with a scalar one. + order_qt_partition_mixed_keys """ + select id, row_number() over (partition by v['k'], id % 2 order by id) as rn + from test_variant_window_partition + """ + + // Once the analytic sink has buffered more than 256 input blocks it erases the rows it has + // consumed from its partition and order key columns. A small batch size reaches that with a + // few thousand rows; the keys are typed Variant (CAST of a number) and encoded Variant + // (parse_to_variant) so both layouts are erased. + sql "set batch_size = 8" + qt_many_blocks_typed_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select cast(number as variant) v, cast(number % 3 as variant) p + from numbers("number" = "5000")) t + ) r + """ + qt_many_blocks_encoded_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select parse_to_variant(cast(number as string)) v, + parse_to_variant(cast(number % 3 as string)) p + from numbers("number" = "5000")) t + ) r + """ + } +} diff --git a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy index 9e8afe7ea14fcb..bd8987819856db 100644 --- a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy +++ b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy @@ -77,23 +77,14 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { contains "EXCEPT" } - test { - sql "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + // V2 canonical equality is supported in scalar and join contexts. + qt_equals "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_not_equals "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_null_safe_equals "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - test { - sql """ + order_qt_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -101,11 +92,8 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v = b.v """ - exception "CAST to a concrete type first" - } - test { - sql """ + order_qt_null_safe_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -113,8 +101,6 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v <=> b.v """ - exception "CAST to a concrete type first" - } order_qt_group_by """ SELECT CAST(v AS STRING), COUNT(*) diff --git a/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md new file mode 100644 index 00000000000000..1c2741280d3081 --- /dev/null +++ b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md @@ -0,0 +1,101 @@ +# VARIANT relational benchmark + +On branch-4.2, the benchmark load and relational suites temporarily enable +Variant V2 and restore the previous FE configuration after each phase. A plain +`load_p2` run keeps the current Variant configuration. + +`relational_performance.groovy` extends the existing `variant_p2` GitHub Events +workload with GROUP BY, ORDER BY, and broadcast/shuffle equality JOIN. The load +phase runs `load.groovy` and verifies its expected 44,273,863 rows. Pass +`--stream-load` to read the same public hourly files over HTTPS when regression +S3 credentials are unavailable. HTTP reads use a socket timeout and resumable +download. If an interrupted run has committed all but a known set of files, +pass those names with `--resume-files`; this appends only those files before the +same exact row-count check and prepare phase. +It then creates one small dimension table per selected real JSON path: + +| Key | Source path | CAST comparison | +| --- | --- | --- | +| actor_login | `actor['login']` | STRING | +| repo_name | `repo['name']` | STRING | +| payload_action | `payload['action']` | STRING | +| actor_id | `actor['id']` | BIGINT | + +The default pair is `actor_login,actor_id`, covering one string and one numeric +path. Use `--keys` to select any subset of the four paths. + +Each query has native VARIANT and explicit CAST forms. SQL cache and query cache +are disabled. GROUP BY correctness uses bidirectional EXCEPT between complete +native and CAST group results before timing compact aggregate checksums. JOIN +checks count and ID sum and uses one dimension row per key to avoid many-to-many +output explosion. + +| Operation | What it measures | +| --- | --- | +| `group` | GROUP BY on the key, reduced to a compact checksum | +| `order_topn` | `ORDER BY key NULLS FIRST, id LIMIT 1000`: a TopN, not a full sort | +| `order_full` | `row_number() OVER (ORDER BY key NULLS FIRST, id)` over every row: a full sort whose `sum(id * rn)` checksum compares the complete native and CAST orders | +| `join_broadcast`, `join_shuffle` | equality join against the dimension table | + +Variant join keys get no runtime filter while CAST keys do. Each join therefore +has three modes: `native`, `cast`, and `cast_no_rf` (the CAST join with +`runtime_filter_mode='OFF'`). Compare `native` with `cast_no_rf` for the key +comparison cost and `cast` with `cast_no_rf` for the runtime filter benefit. + +A join against an empty or stale dimension table would still make native and +CAST agree. Before timing a join, the query phase checks with bidirectional +EXCEPT that the dimension table equals the current grouping of `github_events` +and is not empty; rerun the load phase (or `VARIANT_BENCH_PHASE=prepare`) if it +fails. + +Mixed-type semantics belong in `variant_p0/test_variant_relational_corners.groovy`. +They are deliberately excluded from these performance pairs: CAST may merge +strings with numbers or round integers above 2^53. Matching CAST results alone +cannot prove correct native mixed-type behavior. + +Build and start this worktree's FE/BE using the repository environment workflow. +Performance requires `BUILD_TYPE=RELEASE` and `./build.sh --be --fe`; do not pass +an explicit `-j`. Synchronize worktree ports after building. The runner requires +Python 3.11+, Linux `/proc`, and the local `output/` runtime. Use a dedicated +database/config because `load.groovy` recreates `github_events`. + +Run from the worktree root, substituting the verified regression config: + +```bash +python3 regression-test/suites/variant_p2/run_relational_benchmark.py load \ + --conf tmp/regression-conf.auto.groovy --stream-load \ + --output tmp/relational-load + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --warmups 2 --repeats 7 \ + --output tmp/relational-query + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --keys actor_login \ + --warmups 0 --repeats 3 --spill --output tmp/relational-spill +``` + +Choose eight distinct physical cores after inspecting host topology and load. +The load phase does not change affinity and rejects an already eight-core-bound +FE or BE process. Doris may independently pin a few service threads. The query +phase binds every FE and BE thread to the same eight cores, checks affinity +during the run, and restores previous masks afterward. This restricts the +database servers; the regression client runs outside that mask. CPU affinity +does not reserve cores against unrelated host processes. + +The runner verifies the running BE matches the installed Release binary. Each +new evidence directory contains revision/binary/affinity metadata, host load, +regression logs, query plans, every warmup/measured latency and result SHA-256, +and median/min/max summaries. Latency includes client round-trip and result +consumption. The starting mode rotates each round. Interpret forced-spill +measurements separately from ordinary queries. A completed repeat run checks +repeatability, not long-duration soak, concurrent ingestion, or crash recovery. + +The query suite can also run directly through `run-regression-test.sh` using +`VARIANT_BENCH_PHASE=prepare|query` (required: without it an ordinary `variant_p2` +run skips the suite), `VARIANT_BENCH_ROWS`, `VARIANT_BENCH_KEYS`, +`VARIANT_BENCH_REPEATS`, `VARIANT_BENCH_WARMUPS`, `VARIANT_BENCH_RESULTS`, and +`VARIANT_BENCH_SPILL`. Direct runs do not enforce Release or CPU affinity and +must not be reported as an eight-core performance result. diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 8cb0bc19255f30..b8d26583136334 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -18,162 +18,176 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit suite("load_p2", "variant_type,p2"){ - boolean use_stream_load = false - def load_json_data = {table_name, file_name -> - // load the json data - streamLoad { - table "${table_name}" + def enableVariantV2 = System.getenv().containsKey("VARIANT_BENCH_PHASE") || + getFeConfig("enable_variant_v2").toBoolean() + setFeConfigTemporary([enable_variant_v2: enableVariantV2]) { + def env = System.getenv() + boolean use_stream_load = env + .getOrDefault("VARIANT_P2_USE_STREAM_LOAD", "false").toBoolean() + def resumeFiles = env.getOrDefault("VARIANT_P2_RESUME_FILES", "") + .split(",").findAll { !it.isEmpty() } + def load_json_data = {table_name, file_name -> + // load the json data + streamLoad { + table "${table_name}" - // set http request header params - set 'read_json_by_line', 'true' - set 'format', 'json' - set 'max_filter_ratio', '0.1' - file file_name // import json file - time 10000 // limit inflight 10s + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + set 'max_filter_ratio', '0.1' + file file_name // import json file + time 10000 // limit inflight 10s + timeout 60000 + retryIfHttpError true - // if declared a check callback, the default check condition will ignore. - // So you must check all condition + // if declared a check callback, the default check condition will ignore. + // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Stream load ${file_name} result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } - logger.info("Stream load ${file_name} result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } } - } - def create_table = {table_name, buckets="auto" -> - sql "set default_variant_max_subcolumns_count = 0" - sql "DROP TABLE IF EXISTS ${table_name}" - sql """ - CREATE TABLE IF NOT EXISTS ${table_name} ( - id BIGINT NOT NULL, - type VARCHAR(30) NULL, - actor VARIANT NULL, - repo VARIANT NULL, - payload VARIANT NULL, - public BOOLEAN NULL, - created_at DATETIME NULL, - org JSON NULL - -- INDEX idx_payload(payload) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', - -- INDEX idx_repo(repo) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', - -- INDEX idx_actor(actor) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' - ) - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(id) BUCKETS ${buckets} - properties("replication_num" = "1", "disable_auto_compaction" = "false"); - """ - } + def create_table = {table_name, buckets="auto" -> + sql "set default_variant_max_subcolumns_count = 0" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + id BIGINT NOT NULL, + type VARCHAR(30) NULL, + actor VARIANT NULL, + repo VARIANT NULL, + payload VARIANT NULL, + public BOOLEAN NULL, + created_at DATETIME NULL, + org JSON NULL + -- INDEX idx_payload(payload) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', + -- INDEX idx_repo(repo) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', + -- INDEX idx_actor(actor) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(id) BUCKETS ${buckets} + properties("replication_num" = "1", "disable_auto_compaction" = "false"); + """ + } - // Configuration for the number of threads - def numberOfThreads = 20 // Set this to your desired number of threads + // Configuration for the number of threads + def numberOfThreads = 20 // Set this to your desired number of threads - // Executor service for managing threads - def executorService = Executors.newFixedThreadPool(numberOfThreads) - def futures = [] + // Executor service for managing threads + def executorService = Executors.newFixedThreadPool(numberOfThreads) + def futures = [] - try { - def table_name = "github_events" - def s3load_paral_wait = {tbl, fmt, path, paral -> - String ak = getS3AK() - String sk = getS3SK() - String s3BucketName = getS3BucketName() - String s3Endpoint = getS3Endpoint() - String s3Region = getS3Region() - def load_label = "part_" + UUID.randomUUID().toString().replace("-", "0") - sql """ - LOAD LABEL ${load_label} ( - DATA INFILE("s3://${s3BucketName}/${path}") - INTO TABLE ${tbl} - COLUMNS TERMINATED BY "," - FORMAT AS "${fmt}" - ) - WITH S3 ( - "AWS_ACCESS_KEY" = "$ak", - "AWS_SECRET_KEY" = "$sk", - "AWS_ENDPOINT" = "${s3Endpoint}", - "AWS_REGION" = "${s3Region}", - "provider" = "${getS3Provider()}" - ) - PROPERTIES( - "load_parallelism" = "${paral}" - ); - """ - // Waiting for job finished or cancelled - def max_try_milli_secs = 600000 - while (max_try_milli_secs > 0) { - String[][] result = sql """ show load where label="$load_label" order by createtime desc limit 1; """ - if (result[0][2].equals("FINISHED")) { - logger.info("Load FINISHED " + load_label) - break; - } - if (result[0][2].equals("CANCELLED")) { - log.warn("load failed: $result") - assertTrue(false, "load failed: $result") - break; + try { + def table_name = "github_events" + def s3load_paral_wait = {tbl, fmt, path, paral -> + String ak = getS3AK() + String sk = getS3SK() + String s3BucketName = getS3BucketName() + String s3Endpoint = getS3Endpoint() + String s3Region = getS3Region() + def load_label = "part_" + UUID.randomUUID().toString().replace("-", "0") + sql """ + LOAD LABEL ${load_label} ( + DATA INFILE("s3://${s3BucketName}/${path}") + INTO TABLE ${tbl} + COLUMNS TERMINATED BY "," + FORMAT AS "${fmt}" + ) + WITH S3 ( + "AWS_ACCESS_KEY" = "$ak", + "AWS_SECRET_KEY" = "$sk", + "AWS_ENDPOINT" = "${s3Endpoint}", + "AWS_REGION" = "${s3Region}", + "provider" = "${getS3Provider()}" + ) + PROPERTIES( + "load_parallelism" = "${paral}" + ); + """ + // Waiting for job finished or cancelled + def max_try_milli_secs = 600000 + while (max_try_milli_secs > 0) { + String[][] result = sql """ show load where label="$load_label" order by createtime desc limit 1; """ + if (result[0][2].equals("FINISHED")) { + logger.info("Load FINISHED " + load_label) + break; + } + if (result[0][2].equals("CANCELLED")) { + log.warn("load failed: $result") + assertTrue(false, "load failed: $result") + break; + } + Thread.sleep(6000) + max_try_milli_secs -= 6000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $load_label") + } } - Thread.sleep(6000) - max_try_milli_secs -= 6000 - if(max_try_milli_secs <= 0) { - assertTrue(1 == 2, "load Timeout: $load_label") + } + def submitFile = { fileName -> + if (use_stream_load) { + def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + load_json_data.call(table_name, fileUrl) + } as Runnable) + } else { + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + s3load_paral_wait.call(table_name, "JSON", + "regression/github_events_dataset/${fileName}", 3) + } as Runnable) } } - } - create_table.call(table_name) - List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - // 2015 - def year = "2015" - def monthPrefix = "0" - def dayPrefix = "0" - log.info("current year: ${year}") - for (int i = 1; i <= 3; i++) { - def month = i < 10 ? monthPrefix + i.toString() : i.toString() - log.info("current month: ${month}") - for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { - def day = j < 10 ? dayPrefix + j.toString() : j.toString() - log.info("current day: ${day}") - for (int z = 1; z < 24; z++) { - def hour = z.toString() - log.info("current hour: ${hour}") - def fileName = year + "-" + month + "-" + day + "-" + hour + ".json" - log.info("cuurent fileName: ${fileName}") - if (use_stream_load) { - def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - load_json_data.call(table_name, fileUrl) - } as Runnable) - } else { - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - s3load_paral_wait.call(table_name, "JSON", "regression/github_events_dataset/${fileName}", 3) - } as Runnable) + if (resumeFiles) { + assertTrue(use_stream_load, "Resume requires VARIANT_P2_USE_STREAM_LOAD=true") + resumeFiles.each { submitFile.call(it) } + } else { + create_table.call(table_name) + List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + def year = "2015" + def monthPrefix = "0" + def dayPrefix = "0" + log.info("current year: ${year}") + for (int i = 1; i <= 3; i++) { + def month = i < 10 ? monthPrefix + i.toString() : i.toString() + log.info("current month: ${month}") + for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { + def day = j < 10 ? dayPrefix + j.toString() : j.toString() + log.info("current day: ${day}") + for (int z = 1; z < 24; z++) { + def fileName = year + "-" + month + "-" + day + "-" + z + ".json" + log.info("current fileName: ${fileName}") + submitFile.call(fileName) + } } } } - } - - try { - futures.each { future -> - future.get() + + try { + futures.each { future -> + future.get() + } + } catch (Exception e) { + throw e.cause // throw original exception + } finally { + // Shutdown executor service and wait for all tasks to complete + executorService.shutdown() + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) } - } catch (Exception e) { - throw e.cause // throw original exception + + qt_sql("select count() from github_events") } finally { - // Shutdown executor service and wait for all tasks to complete - executorService.shutdown() - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) + // reset flags } - - qt_sql("select count() from github_events") - } finally { - // reset flags } } diff --git a/regression-test/suites/variant_p2/relational_performance.groovy b/regression-test/suites/variant_p2/relational_performance.groovy new file mode 100644 index 00000000000000..f9355556b9e13b --- /dev/null +++ b/regression-test/suites/variant_p2/relational_performance.groovy @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import java.security.MessageDigest + +suite("variant_relational_performance", "p2,nonConcurrent") { + def env = System.getenv() + // This benchmark is driven by run_relational_benchmark.py. A plain variant_p2 run has no + // prepared dimension tables, so the suite only runs when a phase is set explicitly. + def phase = env.get("VARIANT_BENCH_PHASE") + if (phase == null) { + log.info("Skip variant_relational_performance: VARIANT_BENCH_PHASE is not set") + return + } + long expectedRows = env.getOrDefault("VARIANT_BENCH_ROWS", "44273863").toLong() + int repeats = env.getOrDefault("VARIANT_BENCH_REPEATS", "7").toInteger() + int warmups = env.getOrDefault("VARIANT_BENCH_WARMUPS", "2").toInteger() + if (!(phase in ["prepare", "query"]) || expectedRows < 1 || repeats < 1 || warmups < 0) { + throw new IllegalArgumentException("Invalid VARIANT_BENCH configuration") + } + // These paths come directly from variant_p2/sql and the GitHub Events schema. + def keys = [ + actor_login: [column: "actor", path: "login", type: "STRING"], + repo_name: [column: "repo", path: "name", type: "STRING"], + payload_action: [column: "payload", path: "action", type: "STRING"], + actor_id: [column: "actor", path: "id", type: "BIGINT"] + ] + def requestedKeys = env.getOrDefault("VARIANT_BENCH_KEYS", "actor_login,actor_id").split(",") as Set + if (!keys.keySet().containsAll(requestedKeys)) { + throw new IllegalArgumentException("Unknown VARIANT_BENCH_KEYS: ${requestedKeys - keys.keySet()}") + } + keys = keys.findAll { key, ignored -> requestedKeys.contains(key) } + + setFeConfigTemporary([enable_variant_v2: true]) { + def actualRows = (sql("SELECT count(*) FROM github_events"))[0][0].toString().toLong() + assertEquals(expectedRows, actualRows) + if (phase == "prepare") { + sql "SET default_variant_max_subcolumns_count = 0" + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + sql "DROP TABLE IF EXISTS variant_relational_dim_${key}" + sql """CREATE TABLE variant_relational_dim_${key} (id BIGINT, k VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_dim_${key} + SELECT min(id), ${nativeKey} FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${nativeKey}""" + } + return + } + + sql "SET parallel_pipeline_task_num = 8" + sql "SET enable_sql_cache = false" + sql "SET enable_query_cache = false" + if (env.getOrDefault("VARIANT_BENCH_SPILL", "false").toBoolean()) { + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + } + def output = new File(env.getOrDefault("VARIANT_BENCH_RESULTS", + "tmp/variant-relational-results.jsonl")) + output.parentFile.mkdirs() + def record = { event -> output.append(JsonOutput.toJson(event) + "\n") } + def fingerprint = { result -> + MessageDigest.getInstance("SHA-256").digest(JsonOutput.toJson( + result.collect { row -> row.collect { value -> value == null ? null : value.toString() } } + ).getBytes("UTF-8")).encodeHex().toString() + } + def query = { statement -> + long start = System.nanoTime() + def result = sql(statement) + [ms: (System.nanoTime() - start) / 1e6, hash: fingerprint(result), resultRows: result.size()] + } + record([event: "start", rows: actualRows, repeats: repeats, warmups: warmups, + parallelPipelineTasks: 8, keys: keys.keySet(), cpus: env.get("VARIANT_BENCH_CPUS"), + spill: env.get("VARIANT_BENCH_SPILL"), time: new Date().toString()]) + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + def castKey = "CAST(${nativeKey} AS ${spec.type})" + def nativeGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${nativeKey}""" + def castGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${castKey}""" + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${nativeGroups}) EXCEPT (${castGroups})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${castGroups}) EXCEPT (${nativeGroups})) difference"""))[0][0].toString().toInteger()) + record([event: "full_group_correctness", key: key]) + + // ORDER BY ... LIMIT is a TopN and never sorts the whole input. The window forces a + // full sort of every row, and its order-sensitive checksum compares the complete + // native order with the CAST order instead of the first rows only. + def fullSort = { orderKey -> """SELECT count(*), sum(CAST(id AS LARGEINT) * rn) FROM ( + SELECT id, row_number() OVER (ORDER BY ${orderKey} NULLS FIRST, id) rn + FROM github_events) ranked""" } + def queries = [ + group: [ + native: "SELECT count(*), sum(n*n), sum(first_id) FROM (${nativeGroups}) g", + cast: "SELECT count(*), sum(n*n), sum(first_id) FROM (${castGroups}) g" + ], + order_topn: [ + native: "SELECT id FROM github_events ORDER BY ${nativeKey} NULLS FIRST, id LIMIT 1000", + cast: "SELECT id FROM github_events ORDER BY ${castKey} NULLS FIRST, id LIMIT 1000" + ], + order_full: [native: fullSort(nativeKey), cast: fullSort(castKey)] + ] + + // A join against an empty or stale dimension still makes native and CAST agree, so + // prove the dimension is the current grouping of the source before timing a join. + def dimension = "SELECT id, CAST(k AS ${spec.type}) k FROM variant_relational_dim_${key}" + def expectedDimension = """SELECT min(id) id, ${castKey} k FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${castKey}""" + def dimensionRows = (sql("SELECT count(*) FROM variant_relational_dim_${key}"))[0][0] + .toString().toLong() + assertTrue(dimensionRows > 0) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${dimension}) EXCEPT (${expectedDimension})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${expectedDimension}) EXCEPT (${dimension})) difference"""))[0][0].toString().toInteger()) + record([event: "dimension_verified", key: key, rows: dimensionRows]) + + // Variant keys get no runtime filter while the CAST keys do, so cast_no_rf isolates the + // key comparison cost from the runtime filter benefit. + def leftKey = "l.${spec.column}['${spec.path}']" + def castPredicate = "CAST(${leftKey} AS ${spec.type}) = CAST(r.k AS ${spec.type})" + ["broadcast", "shuffle"].each { distribution -> + def join = { hint, predicate -> """SELECT ${hint} count(*), sum(l.id) FROM github_events l + JOIN [${distribution}] variant_relational_dim_${key} r ON ${predicate}""" } + queries["join_${distribution}".toString()] = [ + native: join("", "${leftKey} = r.k"), + cast: join("", castPredicate), + cast_no_rf: join("/*+ SET_VAR(runtime_filter_mode='OFF') */", castPredicate) + ] + } + + queries.each { operation, statements -> + def oracle = query(statements.cast).hash + statements.each { mode, statement -> + record([event: "plan", key: key, operation: operation, mode: mode, + plan: sql("EXPLAIN ${statement}")]) + } + def modes = statements.keySet() as List + for (int round = -warmups; round < repeats; ++round) { + // Rotate the starting mode so no mode always runs first or last. + Collections.rotate(modes, 1) + modes.each { mode -> + def sample = query(statements[mode]) + assertEquals(oracle, sample.hash) + record(sample + [event: "sample", key: key, operation: operation, + mode: mode, round: round, sql: statements[mode]]) + } + } + } + } + record([event: "complete", time: new Date().toString()]) + } +} diff --git a/regression-test/suites/variant_p2/run_relational_benchmark.py b/regression-test/suites/variant_p2/run_relational_benchmark.py new file mode 100644 index 00000000000000..72e9a19660dd89 --- /dev/null +++ b/regression-test/suites/variant_p2/run_relational_benchmark.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run the regression suite with unrestricted load and verified 8-core queries.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import statistics +import subprocess +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("load", "query")) + parser.add_argument("--conf", required=True) + parser.add_argument("--cpus", help="Eight comma-separated physical CPU IDs (query only)") + parser.add_argument("--rows", type=int, default=44_273_863) + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--keys", default="actor_login,actor_id") + parser.add_argument("--stream-load", action="store_true", + help="Load public variant_p2 files over HTTP instead of authenticated S3") + parser.add_argument("--resume-files", default="", + help="Comma-separated public variant_p2 files to append after an interrupted load") + parser.add_argument("--spill", action="store_true", help="Separate forced-spill correctness/stability run") + parser.add_argument("--output", required=True, help="New evidence directory") + args = parser.parse_args() + if args.phase == "query" and (args.stream_load or args.resume_files): + parser.error("--stream-load and --resume-files apply only to the load phase") + if args.resume_files and not args.stream_load: + parser.error("--resume-files requires --stream-load") + # Job cancellation sends SIGTERM. Unwind through the finally block below so the regression + # child is stopped and FE/BE thread affinity is restored. + signal.signal(signal.SIGTERM, lambda signum, _: sys.exit(128 + signum)) + repo = Path(__file__).resolve().parents[3] + evidence = Path(args.output).resolve() + evidence.mkdir(parents=True, exist_ok=False) + processes = {} + original = {} + for name in ("be", "fe"): + pid = int((repo / f"output/{name}/bin/{name}.pid").read_text()) + command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode() + if str(repo) not in command: + raise RuntimeError(f"Refusing to change unrelated {name} PID {pid}") + processes[name] = pid + original[pid] = {int(t.name): os.sched_getaffinity(int(t.name)) + for t in Path(f"/proc/{pid}/task").iterdir()} + # The regression client must query the FE whose threads are pinned and fingerprinted here. + conf_port = re.search(r"jdbc:mysql://[^:/]+:(\d+)", Path(args.conf).read_text()) + fe_port = re.search(r"^\s*query_port\s*=\s*(\d+)", + (repo / "output/fe/conf/fe.conf").read_text(), re.MULTILINE) + if conf_port is None or int(conf_port.group(1)) != int(fe_port.group(1) if fe_port else 9030): + raise RuntimeError("--conf does not point at this worktree's FE query port") + cpus = None + if args.phase == "query": + cpus = {int(cpu) for cpu in (args.cpus or "").split(",") if cpu} + topology = {tuple((Path(f"/sys/devices/system/cpu/cpu{cpu}/topology") / item) + .read_text().strip() for item in ("physical_package_id", "core_id")) + for cpu in cpus} + if len(cpus) != 8 or len(topology) != 8: + raise ValueError("Select exactly eight distinct physical cores") + cache = (repo / "be/build_RELEASE/CMakeCache.txt").read_text() + if "CMAKE_BUILD_TYPE:STRING=RELEASE" not in cache.upper(): + raise RuntimeError("A Release build is required for performance measurements") + binary = repo / "output/be/lib/doris_be" + running = Path(f"/proc/{processes['be']}/exe") + if not os.path.samefile(binary, running): + raise RuntimeError("Running BE differs from the worktree output binary") + release_binary = repo / "be/build_RELEASE/src/service/doris_be" + with release_binary.open("rb") as built, binary.open("rb") as installed: + if hashlib.file_digest(built, "sha256").digest() != hashlib.file_digest(installed, "sha256").digest(): + raise RuntimeError("Installed BE does not match the Release build") + elif args.cpus: + raise ValueError("Do not bind CPUs during ingestion") + else: + for pid in processes.values(): + if len(os.sched_getaffinity(pid)) <= 8: + raise RuntimeError("Load requires each FE/BE process to be unrestricted beyond eight CPUs") + manifest = dict(vars(args), processes=processes, checkout=str(repo), + head=subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip(), + started=time.time(), original_affinity={str(p): {str(t): sorted(m) for t, m in ts.items()} + for p, ts in original.items()}) + manifest["harness_sha256"] = { + name: hashlib.sha256((Path(__file__).parent / name).read_bytes()).hexdigest() + for name in ("load.groovy", "relational_performance.groovy", Path(__file__).name) + } + manifest["dirty_checkout"] = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], cwd=repo, text=True).strip()) + # Hash what the recorded processes run, not only what the checkout contains. + with Path(f"/proc/{processes['be']}/exe").open("rb") as binary: + manifest["be_sha256"] = hashlib.file_digest(binary, "sha256").hexdigest() + with (repo / "output/fe/lib/doris-fe.jar").open("rb") as jar: + manifest["fe_jar_sha256"] = hashlib.file_digest(jar, "sha256").hexdigest() + (evidence / "manifest.json").write_text(json.dumps(manifest, indent=2)) + environment = dict(os.environ, VARIANT_BENCH_PHASE=args.phase, + VARIANT_BENCH_ROWS=str(args.rows), VARIANT_BENCH_REPEATS=str(args.repeats), + VARIANT_BENCH_WARMUPS=str(args.warmups), + VARIANT_BENCH_KEYS=args.keys, + VARIANT_BENCH_RESULTS=str(evidence / "samples.jsonl"), + VARIANT_BENCH_SPILL=str(args.spill).lower(), + VARIANT_P2_USE_STREAM_LOAD=str(args.stream_load).lower(), + VARIANT_P2_RESUME_FILES=args.resume_files, + VARIANT_BENCH_CPUS=args.cpus or "unrestricted") + for key in list(environment): + if key.lower() in ("http_proxy", "https_proxy", "all_proxy"): + del environment[key] + environment["NO_PROXY"] = environment["no_proxy"] = "127.0.0.1,localhost" + child = None + try: + if cpus: + # Re-enumerate until every thread has the mask. New threads inherit + # the creator's mask; the monitor below rejects any subsequent drift. + for pid in processes.values(): + for _ in range(10): + tids = [int(t.name) for t in Path(f"/proc/{pid}/task").iterdir()] + for tid in tids: + try: + os.sched_setaffinity(tid, cpus) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(t.name)) == cpus + for t in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not establish CPU affinity") + suites = (["load_p2", "variant_relational_performance"] + if args.phase == "load" else ["variant_relational_performance"]) + with (evidence / "host-load.jsonl").open("w") as host: + for suite in suites: + suite_environment = environment.copy() + if args.phase == "load" and suite == "variant_relational_performance": + suite_environment["VARIANT_BENCH_PHASE"] = "prepare" + log_name = ({"load_p2": "load.log"}.get(suite, "prepare.log") + if args.phase == "load" else "regression.log") + command = [str(repo / "run-regression-test.sh"), "--conf", args.conf, "--run", + "-d", "variant_p2", "-s", suite] + with (evidence / log_name).open("w") as log: + child = subprocess.Popen(command, cwd=repo, env=suite_environment, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + while child.poll() is None: + host.write(json.dumps({"time": time.time(), + "loadavg": Path("/proc/loadavg").read_text().strip()}) + "\n") + host.flush() + if cpus: + for pid in processes.values(): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + if os.sched_getaffinity(int(thread.name)) != cpus: + raise RuntimeError(f"CPU affinity drift: {thread}") + except ProcessLookupError: + pass + time.sleep(1) + if child.returncode: + raise RuntimeError(f"Regression failed: inspect {evidence / log_name}") + child = None + finally: + if child is not None and child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + child.wait() + if cpus: + for pid, threads in original.items(): + for _ in range(10): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + os.sched_setaffinity(int(thread.name), threads.get(int(thread.name), threads[pid])) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(thread.name)) == + threads.get(int(thread.name), threads[pid]) + for thread in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not restore CPU affinity") + if args.phase == "query": + events = [json.loads(line) for line in (evidence / "samples.jsonl").read_text().splitlines()] + if not events or events[-1]["event"] != "complete": + raise RuntimeError("Missing completion record") + groups = {} + for event in events: + if event["event"] == "sample" and event["round"] >= 0: + groups.setdefault((event["key"], event["operation"], event["mode"]), []).append(event["ms"]) + summary = [{"key": key, "operation": operation, "mode": mode, "samples_ms": values, + "median_ms": statistics.median(values), "min_ms": min(values), "max_ms": max(values)} + for (key, operation, mode), values in groups.items()] + (evidence / "summary.json").write_text(json.dumps(summary, indent=2)) + print(evidence) + + +if __name__ == "__main__": + main() From 1ec8c151a7ff5da600c639c491d337b0909cb52d Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 12 Oct 2026 12:53:13 +0800 Subject: [PATCH 3/5] [test](regression) Stabilize Variant V1 root and schema-change checks ### What problem does this PR solve? Issue Number: None Related PR: #67675 Problem Summary: The backport forwards the JSONB source NULL mask, so newly written V1 empty objects remain {} instead of exposing the old [] placeholder. Regenerate the two affected TopN expectations and update the comment while retaining the materialization checks. Explicitly insert a TEXT row after schema change because light DROP INDEX completes synchronously and offers no RUNNING interval for the existing double-write loop. ### Release note None ### Check List (For Author) - Test: Regression test: test_topn_lazy_materialize_sparse_variant and V1 double_write_schema_change_with_variant, using ASAN BE/FE - Behavior changed: No (test-only changes) - Does this need documentation: No --- .../variant_p0/test_topn_lazy_materialize_sparse_variant.out | 4 ++-- .../schema_change/test_double_write_when_schema_change.groovy | 2 ++ .../test_topn_lazy_materialize_sparse_variant.groovy | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out index 4afc03b5e2b6b0..34632b5b22de47 100644 --- a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out +++ b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out @@ -6,11 +6,11 @@ 4 -9223372036854775808 [1,2,3] {"k":1} 6 1 [1,2,3] {} 7 0 [1,2,3] [1,2,3] -9 \N {} [] +9 \N {} {} 10 1 {} {"n":7,"word":"hot"} 12 1 {"word":"hot","n":7} \N 13 5777142737451291289 {} {"n":7,"word":"hot"} -15 -1 [1,2,3] [] +15 -1 [1,2,3] {} 16 -906638365906932436 {"word":"hot","n":7} {"k":1} 18 -9223372036854775808 {"word":"hot","n":7} {"n":7,"word":"hot"} 19 -9223372036854775808 {"word":"hot","n":7} [1,2,3] diff --git a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy index a2c1b7f6a7e0f2..aacc7cc396713c 100644 --- a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy +++ b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy @@ -113,6 +113,8 @@ suite("double_write_schema_change_with_variant", "nonConcurrent") { sql """ALTER TABLE ${table_name} drop index idx_var""" double_write.call() + // Light index changes can finish synchronously. Explicitly insert into the TEXT column. + sql insert_sql qt_sql "select v['type'], v['id'], v['created_at'] from ${table_name} where cast(v['id'] as bigint) != 25061216922 order by k, cast(v['id'] as bigint), cast(v['payload']['commits'] as string) limit 10" sql "set enable_two_phase_read_opt = false" diff --git a/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy index 5b4c82c9e8b11d..69e2f94bb44db5 100644 --- a/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy +++ b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy @@ -79,8 +79,8 @@ suite("test_topn_lazy_materialize_sparse_variant", "p0") { ORDER BY pk ASC LIMIT 128 """ - // V1 renders an empty object stored at the Variant root as an empty array. Keep that - // historical assertion on V1 and use typed subpaths for the shared V1/V2 check. + // Check V1 root values, including empty objects. Use typed subpaths for the shared + // V1/V2 check below. if (!enableVariantV2) { qt_topn_lazy_sparse_variant_v1 topnV1Query } From 10c62b3b1e9a2371803546e7fc43b60869ee0b16 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 12 Oct 2026 15:23:11 +0800 Subject: [PATCH 4/5] [test](regression) Serialize suites that change Variant FE configuration ### What problem does this PR solve? Issue Number: None Related PR: #67675 Problem Summary: setFeConfigTemporary changes global FE configuration and restores it without synchronization. Mark the five remaining affected suites nonConcurrent so normal parallel suites cannot observe the temporary Variant V2 setting and configuration scopes cannot overwrite each other's saved values. ### Release note None ### Check List (For Author) - Test: Static audit of all 15 modified suites using setFeConfigTemporary against the regression scheduler; git diff --check. No additional runtime tests for group-only edits; prior runtime evidence predates these scheduling changes. - Behavior changed: Yes (schedule affected suites in the serial phase) - Does this need documentation: No --- regression-test/suites/nereids_p0/join/test_join_on.groovy | 2 +- regression-test/suites/variant_p0/test_variant_bool_path.groovy | 2 +- .../suites/variant_p0/test_variant_json_functions.groovy | 2 +- .../suites/variant_p0/test_variant_window_partition.groovy | 2 +- regression-test/suites/variant_p2/load.groovy | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/regression-test/suites/nereids_p0/join/test_join_on.groovy b/regression-test/suites/nereids_p0/join/test_join_on.groovy index c0b742514c544a..d39105c7a60e06 100644 --- a/regression-test/suites/nereids_p0/join/test_join_on.groovy +++ b/regression-test/suites/nereids_p0/join/test_join_on.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_join_on", "nereids_p0") { +suite("test_join_on", "nereids_p0,nonConcurrent") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" diff --git a/regression-test/suites/variant_p0/test_variant_bool_path.groovy b/regression-test/suites/variant_p0/test_variant_bool_path.groovy index 05d98972267ccf..b3c39d9bd14a67 100644 --- a/regression-test/suites/variant_p0/test_variant_bool_path.groovy +++ b/regression-test/suites/variant_p0/test_variant_bool_path.groovy @@ -19,7 +19,7 @@ // true/false, and equality and grouping see two values. That must hold however the rows reach // storage - in one load, one load per row, spread over several tablets - and must survive a // full compaction. -suite("test_variant_bool_path", "p0") { +suite("test_variant_bool_path", "p0,nonConcurrent") { setFeConfigTemporary([enable_variant_v2: true]) { sql "drop table if exists test_variant_bool_path" diff --git a/regression-test/suites/variant_p0/test_variant_json_functions.groovy b/regression-test/suites/variant_p0/test_variant_json_functions.groovy index eaf546afe18583..52e62690746501 100644 --- a/regression-test/suites/variant_p0/test_variant_json_functions.groovy +++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy @@ -16,7 +16,7 @@ // under the License. // JSON functions accept a Variant argument by casting it to JSON implicitly. -suite("test_variant_json_functions", "p0") { +suite("test_variant_json_functions", "p0,nonConcurrent") { setFeConfigTemporary([enable_variant_v2: true]) { sql "DROP TABLE IF EXISTS test_variant_json_functions" diff --git a/regression-test/suites/variant_p0/test_variant_window_partition.groovy b/regression-test/suites/variant_p0/test_variant_window_partition.groovy index ad479232c62485..0860c2537423e9 100644 --- a/regression-test/suites/variant_p0/test_variant_window_partition.groovy +++ b/regression-test/suites/variant_p0/test_variant_window_partition.groovy @@ -19,7 +19,7 @@ // canonical equality and comparison. Window PARTITION BY used to be the one remaining context that // rejected it, so these queries pin that a Variant partition key groups rows by canonical value: // values of different JSON kinds ("a", true, 1) never share a partition, and equal values do. -suite("test_variant_window_partition", "p0") { +suite("test_variant_window_partition", "p0,nonConcurrent") { setFeConfigTemporary([enable_variant_v2: true]) { sql "drop table if exists test_variant_window_partition" diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index b8d26583136334..2e1f8b593527fb 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -17,7 +17,7 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -suite("load_p2", "variant_type,p2"){ +suite("load_p2", "variant_type,p2,nonConcurrent"){ def enableVariantV2 = System.getenv().containsKey("VARIANT_BENCH_PHASE") || getFeConfig("enable_variant_v2").toBoolean() setFeConfigTemporary([enable_variant_v2: enableVariantV2]) { From 164caa4c6b83555bd3f092b46062ad29171364b4 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 12 Oct 2026 22:44:24 +0800 Subject: [PATCH 5/5] [test](variant) Update V1 byte-size expectation after NULL payload masking ### What problem does this PR solve? Issue Number: None Related PR: #67675 Problem Summary: ColumnVariantTest.byte_size loads mixed JSON values and finalizes the V1 column. The backported JSONB source NULL mask skips masked payloads, so the current column uses 11295888 bytes instead of the checked-in 11295900. Regenerate this single expected file using run-be-ut.sh --gen_out while retaining the exact byte-size assertion and NULL-mask protection. ### Release note None ### Check List (For Author) - Test: ASAN BE unit tests: ColumnVariantTest.* and DataTypeVariantV2SerdeJsonbTest.*; byte_size reproduced before regeneration - Behavior changed: No (test expectation only) - Does this need documentation: No --- .../expected_result/vec/columns/column_variant_byte_size.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/test/expected_result/vec/columns/column_variant_byte_size.out b/be/test/expected_result/vec/columns/column_variant_byte_size.out index 7e37b506ec5714..3ac94e60f77efd 100644 --- a/be/test/expected_result/vec/columns/column_variant_byte_size.out +++ b/be/test/expected_result/vec/columns/column_variant_byte_size.out @@ -1,2 +1,2 @@ -column: variant with byte_size: 11295900 -11295900 \ No newline at end of file +column: variant with byte_size: 11295888 +11295888 \ No newline at end of file