diff --git a/be/src/core/column/variant_v2/column_variant_v2.cpp b/be/src/core/column/variant_v2/column_variant_v2.cpp index e3d54a1f19596d..48ee53a8f9f660 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.cpp +++ b/be/src/core/column/variant_v2/column_variant_v2.cpp @@ -44,6 +44,7 @@ #include "core/value/variant/variant_canonical.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/sort/sort_block.h" namespace doris { namespace { @@ -671,6 +672,34 @@ const IColumn& ColumnVariantV2::typed_column() const { return *_typed; } +void ColumnVariantV2::validate_typed_rows() const { + DORIS_CHECK(_typed != nullptr) << "validate_typed_rows requires ColumnVariantV2 typed state"; + switch (_typed_type->get_primitive_type()) { + // Every value of these types maps to a Variant scalar, so there is nothing to reject and no + // reason to format each LARGEINT or IP address only to discard the text. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_TIMESTAMP_NS: + case TYPE_IPV4: + case TYPE_IPV6: + return; + default: + break; + } + // with_variant_typed_scalar() is the one definition of what Variant can encode: it throws for + // an invalid date, a string that is not UTF-8 or a decimal outside its width. A type missing + // from the list above is validated, which is the safe side. + visit_typed_scalar_column(assert_cast(*_typed), + _typed_type->get_primitive_type(), _typed_type->get_scale(), 0, + _typed->size(), [](size_t, const VariantScalarRef&) {}); +} + const DataTypePtr& ColumnVariantV2::typed_type() const { DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state"; return _typed_type; @@ -1037,6 +1066,133 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row) const { return {.metadata = {.data = metadata.data, .size = metadata.size}, .value = value}; } +std::pair ColumnVariantV2::_typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const { + if (!is_typed() || !right.is_typed() || + !(_typed_type == right._typed_type || _typed_type->equals(*right._typed_type))) { + return {nullptr, nullptr}; + } + // Admission is a whitelist: a typed type takes the native path only after its native order has + // been checked against the canonical Variant order (scalar_compare in variant_canonical.cpp). + // The premises shared by every admitted type: + // - both sides have the same typed type including its parameters, so DECIMAL scales and + // DATETIME precisions match and native values are directly comparable; + // - every non-null typed row is a value Variant can encode (the rule and who enforces it + // are at create_typed()), so the native order does not rank a value the canonical path + // would reject, such as an invalid date or a string that is not UTF-8; + // - a typed NULL is the Variant null, which the callers order first, as the canonical order + // does. + // A type that is not listed falls back to the canonical comparison, which is always correct. + switch (_typed_type->get_primitive_type()) { + // Exact numerics and temporal values: both orders are the numeric/chronological order. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DATE: + case TYPE_DATEV2: + case TYPE_DATETIME: + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMP_NS: + case TYPE_TIMESTAMPTZ: + // Strings: both orders compare the bytes as unsigned. + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + break; + // Canonical order puts every NaN last and equal to each other, and -0 equal to +0. compare_at() + // of a floating point column orders the same way; its compare_internal() uses < and >, which + // leave NaN unordered. + case TYPE_FLOAT: + case TYPE_DOUBLE: + if (!allow_floating) { + return {nullptr, nullptr}; + } + break; + // Not admitted: LARGEINT magnitudes above 10^38 - 1 are Variant strings, and IPv4/IPv6 are + // Variant strings whose byte order is not the address order. + default: + return {nullptr, nullptr}; + } + return {&assert_cast(typed_column()), + &assert_cast(right.typed_column())}; +} + +int ColumnVariantV2::compare_at(size_t n, size_t m, const IColumn& rhs, + int nan_direction_hint) const { + const auto& right = assert_cast(rhs); + DCHECK_LT(n, size()); + DCHECK_LT(m, right.size()); + + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, true); + left_nullable != nullptr) { + if (left_nullable->is_null_at(n)) { + return right_nullable->is_null_at(m) ? 0 : -1; + } + if (right_nullable->is_null_at(m)) { + return 1; + } + return left_nullable->get_nested_column().compare_at( + n, m, right_nullable->get_nested_column(), nan_direction_hint); + } + + // A column is in exactly one of three states. Encoded rows are canonical Variant bytes that + // get_value_ref() reads in place. A shredded column has no row bytes until its state + // materializes them; get_value_ref() asks the state for them, which builds and caches them on + // the first call. A typed row is a scalar, compared below without being encoded: a sort compares + // O(n log n) times, so nothing here may allocate or encode per comparison. + if (!is_typed() && !right.is_typed()) { + return canonical_compare(get_value_ref(n), right.get_value_ref(m)); + } + if (!is_typed()) { + // Keep the typed side on the left; canonical_compare() is antisymmetric. + return -right.compare_at(m, n, *this, nan_direction_hint); + } + // The typed fast path declined (different typed types, a type outside its whitelist, or an + // encoded/shredded right side). A typed null row is the canonical null scalar, which sorts + // first - the same order the typed fast path applies. + int result = 0; + const auto& left_nullable = + assert_cast(typed_column()); + visit_typed_scalar_column( + left_nullable, _typed_type->get_primitive_type(), _typed_type->get_scale(), n, n + 1, + [&](size_t, const VariantScalarRef& left_scalar) { + if (!right.is_typed()) { + result = canonical_compare(left_scalar, right.get_value_ref(m)); + return; + } + visit_typed_scalar_column( + assert_cast( + right.typed_column()), + right._typed_type->get_primitive_type(), right._typed_type->get_scale(), m, + m + 1, [&](size_t, const VariantScalarRef& right_scalar) { + result = canonical_compare(left_scalar, right_scalar); + }); + }); + return result; +} + +void ColumnVariantV2::compare_internal(size_t rhs_row_id, const IColumn& rhs, + int nan_direction_hint, int direction, + std::vector& cmp_res, + uint8_t* __restrict filter) const { + const auto& right = assert_cast(rhs); + // ColumnVector::compare_internal does not provide Variant's canonical NaN ordering, so floating + // point typed columns take the canonical path below. + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, false); + left_nullable != nullptr) { + left_nullable->compare_internal(rhs_row_id, *right_nullable, -1, direction, cmp_res, + filter); + return; + } + IColumn::compare_internal(rhs_row_id, rhs, nan_direction_hint, direction, cmp_res, filter); +} + Field ColumnVariantV2::operator[](size_t row) const { Field result; get(row, result); @@ -1465,6 +1621,29 @@ void ColumnVariantV2::pop_back(size_t length) { _check_invariants(); } +void ColumnVariantV2::erase(size_t start, size_t length) { + DORIS_CHECK_LE(start, size()) << "erase start exceeds the column size"; + DORIS_CHECK_LE(length, size() - start) << "erase range exceeds the column size"; + if (length == 0) { + return; + } + if (_shredded) { + ensure_encoded(); + } + if (_typed) { + mutate_subcolumn(_typed); + _typed->erase(start, length); + _check_invariants(); + return; + } + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + // The metadata dictionary keeps blobs that no remaining row references; ids stay valid. + _values->erase(start, length); + _meta_ids->erase(start, length); + _check_invariants(); +} + StringRef ColumnVariantV2::get_data_at(size_t) const { throw_unsupported("get_data_at"); } @@ -1602,6 +1781,21 @@ void ColumnVariantV2::serialize(StringRef* keys, size_t num_rows) const { } } +void ColumnVariantV2::serialize_with_nullable(StringRef* keys, size_t num_rows, bool has_null, + const uint8_t* __restrict null_map) const { + if (has_null) { + IColumn::serialize_with_nullable(keys, num_rows, has_null, null_map); + return; + } + // Without nulls every row carries the same zero flag, so write the flags first and let + // serialize() append each value at the advanced cursor. + for (size_t row = 0; row < num_rows; ++row) { + *(const_cast(keys[row].data) + keys[row].size) = 0; + keys[row].size += sizeof(UInt8); + } + serialize(keys, num_rows); +} + void ColumnVariantV2::deserialize(StringRef* keys, size_t num_rows) { DCHECK(keys != nullptr || num_rows == 0); for (size_t row = 0; row < num_rows; ++row) { @@ -1945,8 +2139,32 @@ void ColumnVariantV2::resize(size_t new_size) { } } -void ColumnVariantV2::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const { - throw_unsupported("get_permutation"); +void ColumnVariantV2::get_permutation(bool reverse, size_t limit, int nan_direction_hint, + HybridSorter& sorter, Permutation& result) const { + const size_t row_count = size(); + result.resize(row_count); + for (size_t row = 0; row < row_count; ++row) { + result[row] = row; + } + + const auto less = [&](size_t left, size_t right) { + const int comparison = compare_at(left, right, *this, nan_direction_hint); + return reverse ? comparison > 0 : comparison < 0; + }; + // std::partial_sort only pays off when limit << row_count, the same threshold ColumnVector uses. + if (static_cast(limit) > static_cast(row_count) / 8.0) { + limit = 0; + } + if (limit != 0 && limit < row_count) { + std::partial_sort(result.begin(), result.begin() + limit, result.end(), less); + } else { + sorter.sort(result.begin(), result.end(), less); + } +} + +void ColumnVariantV2::sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const { + sorter->sort_column(*this, flags, perms, range, last_column); } void ColumnVariantV2::replace_column_data(const IColumn&, size_t, size_t) { diff --git a/be/src/core/column/variant_v2/column_variant_v2.h b/be/src/core/column/variant_v2/column_variant_v2.h index cd9ec2f32a35ab..351b6b28a508c1 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.h +++ b/be/src/core/column/variant_v2/column_variant_v2.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "common/exception.h" #include "core/assert_cast.h" @@ -157,7 +158,21 @@ class ColumnVariantV2 final : public COWHelper { // The input must be an exact, non-Const ColumnNullable whose nested column matches the // non-nullable supported scalar type. + // + // Every row that is non-null in the typed column's own null map must be a value Variant can + // encode. That null map is the only mask every consumer honours: hashing, for one, maps a row + // to a scalar before it looks at an outer null map. Nothing is checked here, because validity + // is enforced where a row becomes a scalar (with_variant_typed_scalar() throws), which covers + // hashing, serialization, encoding and the canonical comparison. The native comparison paths + // do not go through that mapping and rely on the rule instead: + // - producers that decode stored or transported Variant data build valid rows; an external + // Parquet file is trusted to follow its specification, and a violation surfaces as an + // error when the row is hashed or encoded; + // - CAST from a scalar takes arbitrary values, so it folds its null map into the typed one + // and calls validate_typed_rows(). static MutablePtr create_typed(ColumnPtr column, DataTypePtr scalar_type); + // Throws unless every non-null typed row is a value Variant can encode. + void validate_typed_rows() const; static MutablePtr create_shredded(std::shared_ptr state); bool is_typed() const noexcept { return _typed != nullptr; } @@ -198,6 +213,11 @@ class ColumnVariantV2 final : public COWHelper { // including insert, clear, COW mutation, or future row transformations, may invalidate it. VariantRef get_value_ref(size_t row) const; + int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override; + void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint, + int direction, std::vector& cmp_res, + uint8_t* __restrict filter) const override; + Field operator[](size_t row) const override; void get(size_t row, Field& result) const override; void insert(const Field& field) override; @@ -210,6 +230,8 @@ class ColumnVariantV2 final : public COWHelper { void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, const uint32_t* indices_end) override; void pop_back(size_t length) override; + // The analytic sink erases the rows it has consumed from its partition and order key columns. + void erase(size_t start, size_t length) override; StringRef get_data_at(size_t row) const override; void insert_data(const char* pos, size_t length) override; @@ -221,6 +243,8 @@ class ColumnVariantV2 final : public COWHelper { size_t deserialize_impl(const char* pos) override; size_t get_max_row_byte_size() const override; void serialize(StringRef* keys, size_t num_rows) const override; + void serialize_with_nullable(StringRef* keys, size_t num_rows, bool has_null, + const uint8_t* __restrict null_map) const override; void deserialize(StringRef* keys, size_t num_rows) override; void update_hash_with_value(size_t row, SipHash& hash) const override; @@ -247,6 +271,8 @@ class ColumnVariantV2 final : public COWHelper { void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter, Permutation& result) const override; + void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const override; void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override; private: @@ -262,6 +288,11 @@ class ColumnVariantV2 final : public COWHelper { void _adopt_state_from(ColumnVariantV2& replacement); void _detach_metadata_for_write(); void _check_invariants() const; + // Typed columns of both sides when their shared typed type orders natively the same way the + // canonical Variant order does, {nullptr, nullptr} otherwise. allow_floating is false for + // callers that cannot apply Variant's canonical NaN ordering. + std::pair _typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const; void mutate_subcolumns() override; // Encoded state: each row owns a value and references one deduplicated metadata blob. The diff --git a/be/src/core/data_type/data_type_variant_v2.cpp b/be/src/core/data_type/data_type_variant_v2.cpp index 92b0fb4cbd209b..456bc3884483d9 100644 --- a/be/src/core/data_type/data_type_variant_v2.cpp +++ b/be/src/core/data_type/data_type_variant_v2.cpp @@ -48,10 +48,10 @@ MutableColumnPtr DataTypeVariantV2::create_column() const { } bool DataTypeVariantV2::equals(const IDataType& rhs) const { - const auto* rhs_type = typeid_cast(&rhs); - return rhs_type != nullptr && - _max_subcolumns_count == rhs_type->variant_max_subcolumns_count() && - _enable_doc_mode == rhs_type->enable_doc_mode(); + // The max subcolumns count and doc mode only guide how storage shreds a Variant column. + // Execution compares, hashes and serializes Variant values canonically without them, so two + // Variant types with different properties are the same execution type (e.g. hash join keys). + return typeid_cast(&rhs) != nullptr; } int64_t DataTypeVariantV2::get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index d2a262a5648c35..c3980f58821ee5 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -229,7 +229,8 @@ Status DataTypeArraySerDe::serialize_one_cell_to_hive_text( } Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& data_column = assert_cast(from_column); const auto& offsets = data_column.get_offsets(); @@ -242,7 +243,7 @@ Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, } for (size_t i = start; i < end; ++i) { - RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer)); + RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer, options)); } if (!writer.writeEndArray()) { return Status::InternalError("writeEndArray failed"); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index eaf3644efc6879..31378a61081ede 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -122,7 +122,8 @@ class DataTypeArraySerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 9fc8e674d2715b..bb136460743004 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -1255,8 +1255,8 @@ void DataTypeDecimalSerDe::to_string_batch(const IColumn& column, ColumnStrin template Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb"); } else { @@ -1271,7 +1271,7 @@ Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_co template Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb_vector"); diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 04c55a06bf0748..e9f7db67e0b820 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -90,10 +90,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp index e53e4d23a748c8..f2eeaa6c2190f1 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp @@ -265,7 +265,8 @@ Status DataTypeJsonbSerDe::read_column_from_pb(IColumn& column, const PValues& a } Status DataTypeJsonbSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& jsonb_binary = assert_cast(from_column).get_data_at(row_num); const JsonbDocument* doc = nullptr; RETURN_IF_ERROR( diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.h b/be/src/core/data_type_serde/data_type_jsonb_serde.h index 8d14c6d254a868..b97de7fac77b51 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.h +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.h @@ -75,7 +75,8 @@ class DataTypeJsonbSerDe : public DataTypeStringSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index f9814a4b35023c..eb5e16f07700ad 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -766,7 +766,8 @@ Status DataTypeMapSerDe::from_string_strict_mode(StringRef& str, IColumn& column } Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& map_column = assert_cast(from_column); const ColumnArray::Offsets64& offsets = map_column.get_offsets(); @@ -800,7 +801,7 @@ Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, i return Status::InternalError("writeKey failed : {}", key_str.to_string()); } // write value - RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer)); + RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index 158f2620cd26eb..bd6560c053c102 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -117,7 +117,8 @@ class DataTypeMapSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {key_serde, value_serde}; diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index 692e21eac057c0..e354d423f0b2f6 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -123,14 +123,15 @@ Status DataTypeNullableSerDe::serialize_one_cell_to_hive_text( } Status DataTypeNullableSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& col_null = assert_cast(from_column); if (col_null.is_null_at(row_num)) { writer.writeNull(); } else { RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(col_null.get_nested_column(), - row_num, writer)); + row_num, writer, options)); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index d24d110f57313c..f5ec2d1973f6e5 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -124,7 +124,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 5bc68bac04f138..a4b1d2b7cc49ec 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -1187,8 +1187,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) { template Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } @@ -1202,7 +1202,7 @@ Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_col template Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 78d47c499a1457..d067ae720a9866 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -88,10 +88,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index d3844c441de697..93868b304fe6cf 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -145,7 +145,7 @@ Status DataTypeSerDe::default_from_string(StringRef& str, IColumn& column) const } Status DataTypeSerDe::serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map) const { const auto size = from_column.size(); JsonbWriter writer; @@ -155,7 +155,7 @@ Status DataTypeSerDe::serialize_column_to_jsonb_vector( continue; } writer.reset(); - RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer)); + RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer, options)); to_column.insert_data(writer.getOutput()->getBuffer(), writer.getOutput()->getSize()); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index b305f8df79daa7..ffed1008cd3345 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -441,13 +441,14 @@ class DataTypeSerDe { } virtual Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } // Skip rows in source_null_map and insert a default value for each skipped row. virtual Status serialize_column_to_jsonb_vector( - const IColumn& from_column, ColumnString& to_column, + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, const NullMap::value_type* source_null_map = nullptr) const; virtual Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 20286768424ef7..62e5e3da1ae408 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -761,9 +761,9 @@ Status DataTypeStringSerDeBase::write_column_to_orc( } template -Status DataTypeStringSerDeBase::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { +Status DataTypeStringSerDeBase::serialize_column_to_jsonb( + const IColumn& from_column, int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!std::is_same_v) { return Status::NotSupported( "DataTypeStringSerDeBase only supports ColumnString for serialize_column_to_jsonb"); diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index e7c5a9d9c67fd9..df65f438fe7ebe 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -178,7 +178,8 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index a204d6e3d084b5..a295dc2799fc93 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -386,7 +386,8 @@ Status DataTypeStructSerDe::serialize_one_cell_to_hive_text( } Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& struct_column = assert_cast(from_column); if (!writer.writeStartObject()) { @@ -404,7 +405,7 @@ Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column } // write value RETURN_IF_ERROR(elem_serdes_ptrs[i]->serialize_column_to_jsonb(struct_column.get_column(i), - row_num, writer)); + row_num, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index 4873307be18465..1cfa3ee6c74db7 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -113,7 +113,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp index f8a274851de4b9..0a343aa7fa1707 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp @@ -32,6 +32,7 @@ #include "core/arena.h" #include "core/assert_cast.h" #include "core/column/column_const.h" +#include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/column/variant_v2/column_variant_v2_typed_column.h" @@ -46,6 +47,7 @@ #include "core/value/variant/variant_batch_builder.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" +#include "util/jsonb_document.h" #include "util/jsonb_writer.h" #include "util/mysql_row_buffer.h" @@ -565,6 +567,47 @@ void DataTypeVariantV2SerDe::read_one_cell_from_jsonb(IColumn& column, destination(column).insert_encoded_batch(block); } +// to_json and CAST( AS JSON) write Variant values here, with the same session time zone +// as CAST( AS JSON). +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb(const IColumn& from_column, + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { + RETURN_IF_CATCH_EXCEPTION({ + const size_t row = checked_row(row_num); + JsonbWriter document; + visit_variant_v2_values( + from_column, row, row + 1, {}, [](size_t) {}, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, document, {.timezone = options.timezone}); + }); + const JsonbDocument* doc = nullptr; + THROW_IF_ERROR(JsonbDocument::checkAndCreateDocument( + document.getOutput()->getBuffer(), document.getOutput()->getSize(), &doc)); + require_jsonb_write(writer.writeValue(doc->getValue()), "Variant value"); + }); + return Status::OK(); +} + +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { + RETURN_IF_CATCH_EXCEPTION({ + JsonbWriter writer; + visit_variant_v2_values( + from_column, 0, from_column.size(), + source_null_map + ? std::span(source_null_map, from_column.size()) + : std::span(), + [&](size_t) { to_column.insert_default(); }, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, writer, {.timezone = options.timezone}); + to_column.insert_data(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()); + }); + }); + return Status::OK(); +} + namespace { DorisVector json_lengths(const IColumn& column, size_t start, size_t end, diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.h b/be/src/core/data_type_serde/data_type_variant_v2_serde.h index 23a3dcaa941406..d47b40dc43be09 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.h +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.h @@ -53,6 +53,13 @@ class DataTypeVariantV2SerDe final : public DataTypeSerDe { const FormatOptions& options) const override; void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; + Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, + JsonbWriter& writer, + const FormatOptions& options) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; diff --git a/be/src/core/value/variant/variant_canonical.cpp b/be/src/core/value/variant/variant_canonical.cpp index 07b8dcc5965fc6..ffa0dcae0c8cc8 100644 --- a/be/src/core/value/variant/variant_canonical.cpp +++ b/be/src/core/value/variant/variant_canonical.cpp @@ -30,9 +30,12 @@ #include #include +#include "common/compare.h" #include "common/exception.h" +#include "core/uint128.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/common/int_exp.h" #include "exec/common/sip_hash.h" #include "util/hash_util.hpp" #include "util/utf8_check.h" @@ -256,77 +259,253 @@ ObjectEntry object_entry_at(VariantRef object, uint32_t index, StringRef previou return {.key = key, .value = child}; } -bool scalar_equals(const NormalizedValue& left, const NormalizedValue& right) { +bool is_numeric(CanonicalKind kind) { + return kind == CanonicalKind::EXACT_INTEGER || kind == CanonicalKind::DECIMAL || + kind == CanonicalKind::FLOATING; +} + +struct ExactNumber { + unsigned __int128 magnitude; + uint8_t scale; + bool negative; +}; + +ExactNumber exact_number(const NormalizedValue& value) { + DCHECK(value.kind == CanonicalKind::EXACT_INTEGER || value.kind == CanonicalKind::DECIMAL); + return {.magnitude = variant_unsigned_magnitude(value.integer), + .scale = static_cast(value.kind == CanonicalKind::DECIMAL ? value.scale : 0), + .negative = value.integer < 0}; +} + +// A Variant decimal scale is validated to [0, 38] before the value is stored +// (variant_scalar.cpp, variant_value.cpp), so the shared exp10 table, which covers 0 to 76, always +// holds these exponents. +UInt256 power_of_five(uint8_t exponent) { + // 10^k = 2^k * 5^k, so 5^k is the table entry shifted right by k, exactly. + return static_cast(common::exp10_i256(exponent)) >> exponent; +} + +UInt256 power_of_ten(uint8_t exponent) { + return static_cast(common::exp10_i256(exponent)); +} + +uint16_t bit_width(const UInt256& value) { + constexpr unsigned LIMB_BITS = sizeof(value.items[0]) * 8; + // big(index) walks the limbs most significant first, so the first non-zero one carries the + // width; scanning limbs costs 4 iterations instead of up to 256 single-bit shifts. + for (unsigned index = 0; index < UInt256::item_count; ++index) { + const auto limb = value.items[UInt256::_impl::big(index)]; + if (limb != 0) { + return static_cast((UInt256::item_count - index) * LIMB_BITS - + static_cast(std::countl_zero(limb))); + } + } + return 0; +} + +int compare_exact_magnitudes(const ExactNumber& left, const ExactNumber& right) { + // Equal scales are the common case - two JSON integers both have scale 0 - and need no + // rescaling, so they never reach the 256-bit path below. + if (left.scale == right.scale) { + return Compare::compare(left.magnitude, right.magnitude); + } + UInt256 scaled_left {left.magnitude}; + UInt256 scaled_right {right.magnitude}; + if (left.scale < right.scale) { + scaled_left *= power_of_ten(right.scale - left.scale); + } else { + scaled_right *= power_of_ten(left.scale - right.scale); + } + return Compare::compare(scaled_left, scaled_right); +} + +struct FloatingNumber { + uint64_t significand; + int16_t binary_exponent; + bool negative; + bool infinity; + bool nan; +}; + +FloatingNumber floating_number(uint64_t bits) { + constexpr uint64_t FRACTION_MASK = (uint64_t {1} << 52) - 1; + constexpr uint16_t EXPONENT_MASK = (uint16_t {1} << 11) - 1; + const auto exponent = static_cast((bits >> 52) & EXPONENT_MASK); + const uint64_t fraction = bits & FRACTION_MASK; + if (exponent == EXPONENT_MASK) { + return {.significand = 0, + .binary_exponent = 0, + .negative = (bits >> 63) != 0, + .infinity = fraction == 0, + .nan = fraction != 0}; + } + if (exponent == 0) { + return {.significand = fraction, + .binary_exponent = -1074, + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; + } + return {.significand = (uint64_t {1} << 52) | fraction, + .binary_exponent = static_cast(static_cast(exponent) - 1023 - 52), + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; +} + +int compare_floating_exact_magnitudes(const FloatingNumber& floating, const ExactNumber& exact) { + DCHECK(!floating.nan && !floating.infinity); + DCHECK_NE(floating.significand, 0); + DCHECK(exact.magnitude != 0); + UInt256 floating_scaled = UInt256(floating.significand) * power_of_five(exact.scale); + UInt256 exact_scaled {exact.magnitude}; + const int16_t binary_shift = floating.binary_exponent + exact.scale; + if (binary_shift >= 0) { + const uint16_t floating_width = bit_width(floating_scaled); + if (floating_width + binary_shift > bit_width(exact_scaled)) { + return 1; + } + floating_scaled <<= binary_shift; + } else { + const uint16_t exact_width = bit_width(exact_scaled); + const auto shift = static_cast(-binary_shift); + if (exact_width + shift > bit_width(floating_scaled)) { + return -1; + } + exact_scaled <<= shift; + } + return Compare::compare(floating_scaled, exact_scaled); +} + +int compare_numeric(const NormalizedValue& left, const NormalizedValue& right) { + if (left.kind != CanonicalKind::FLOATING && right.kind != CanonicalKind::FLOATING) { + const ExactNumber exact_left = exact_number(left); + const ExactNumber exact_right = exact_number(right); + if (exact_left.negative != exact_right.negative) { + return exact_left.negative ? -1 : 1; + } + int result = compare_exact_magnitudes(exact_left, exact_right); + return exact_left.negative ? -result : result; + } + if (left.kind == CanonicalKind::FLOATING && right.kind == CanonicalKind::FLOATING) { + const FloatingNumber floating_left = floating_number(left.floating_bits); + const FloatingNumber floating_right = floating_number(right.floating_bits); + if (floating_left.nan || floating_right.nan) { + return Compare::compare(floating_left.nan, floating_right.nan); + } + const auto value_left = std::bit_cast(left.floating_bits); + const auto value_right = std::bit_cast(right.floating_bits); + return Compare::compare(value_left, value_right); + } + + const bool floating_on_left = left.kind == CanonicalKind::FLOATING; + const FloatingNumber floating = + floating_number((floating_on_left ? left : right).floating_bits); + const ExactNumber exact = exact_number(floating_on_left ? right : left); + int result; + if (floating.nan) { + result = 1; + } else if (floating.infinity) { + result = floating.negative ? -1 : 1; + } else if (exact.magnitude == 0) { + result = floating.negative ? -1 : 1; + } else if (floating.negative != exact.negative) { + result = floating.negative ? -1 : 1; + } else { + result = compare_floating_exact_magnitudes(floating, exact); + if (floating.negative) { + result = -result; + } + } + return floating_on_left ? result : -result; +} + +int scalar_compare(const NormalizedValue& left, const NormalizedValue& right) { + if (is_numeric(left.kind) && is_numeric(right.kind)) { + const int numeric_result = compare_numeric(left, right); + if (numeric_result != 0) { + return numeric_result; + } + return Compare::compare(left.kind, right.kind); + } + if (left.kind != right.kind) { + return Compare::compare(left.kind, right.kind); + } switch (left.kind) { case CanonicalKind::NULL_VALUE: - return true; + return 0; case CanonicalKind::BOOL: - return left.boolean == right.boolean; + return Compare::compare(left.boolean, right.boolean); case CanonicalKind::EXACT_INTEGER: case CanonicalKind::DATE: case CanonicalKind::TIMESTAMP_TZ: case CanonicalKind::TIMESTAMP_NTZ: case CanonicalKind::TIME: - return left.integer == right.integer; + return Compare::compare(left.integer, right.integer); case CanonicalKind::DECIMAL: - return left.integer == right.integer && left.scale == right.scale; + DCHECK(false) << "Decimal should use numeric comparison"; + return 0; case CanonicalKind::FLOATING: - return left.floating_bits == right.floating_bits; + DCHECK(false) << "Floating point should use numeric comparison"; + return 0; case CanonicalKind::STRING: case CanonicalKind::BINARY: - return left.bytes == right.bytes; + return Compare::compare(left.bytes, right.bytes); case CanonicalKind::UUID: - return left.uuid == right.uuid; + return Compare::compare(left.uuid, right.uuid); case CanonicalKind::OBJECT: case CanonicalKind::ARRAY: break; } - DCHECK(false) << "Container reached scalar equality"; - return false; + DCHECK(false) << "Container reached scalar comparison"; + return 0; } -bool equals_node(VariantRef left, VariantRef right, uint32_t depth) { +int compare_node(VariantRef left, VariantRef right, uint32_t depth) { require_depth(depth); require_exact_value(left); require_exact_value(right); const NormalizedValue normalized_left = normalize_value(left); const NormalizedValue normalized_right = normalize_value(right); if (normalized_left.kind != normalized_right.kind) { - return false; + if (!is_numeric(normalized_left.kind) || !is_numeric(normalized_right.kind)) { + return Compare::compare(normalized_left.kind, normalized_right.kind); + } } if (normalized_left.kind == CanonicalKind::ARRAY) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } - for (uint32_t index = 0; index < count; ++index) { - if (!equals_node(left.array_at(index), right.array_at(index), depth + 1)) { - return false; + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { + const int result = compare_node(left.array_at(index), right.array_at(index), depth + 1); + if (result != 0) { + return result; } } - return true; + return Compare::compare(left_count, right_count); } if (normalized_left.kind == CanonicalKind::OBJECT) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); StringRef previous_left; StringRef previous_right; - for (uint32_t index = 0; index < count; ++index) { + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { const ObjectEntry left_entry = object_entry_at(left, index, previous_left, index != 0); const ObjectEntry right_entry = object_entry_at(right, index, previous_right, index != 0); - if (left_entry.key != right_entry.key || - !equals_node(left_entry.value, right_entry.value, depth + 1)) { - return false; + const int key_result = Compare::compare(left_entry.key, right_entry.key); + if (key_result != 0) { + return key_result; + } + const int value_result = compare_node(left_entry.value, right_entry.value, depth + 1); + if (value_result != 0) { + return value_result; } previous_left = left_entry.key; previous_right = right_entry.key; } - return true; + return Compare::compare(left_count, right_count); } - return scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); } template @@ -355,7 +534,7 @@ void update_bytes(Sink& sink, StringRef bytes) { } template -void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { +inline void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { switch (normalized.kind) { case CanonicalKind::NULL_VALUE: return; @@ -1004,7 +1183,11 @@ void VariantCrc32cHashSink::update(const char* data, size_t size) { } bool canonical_equals(VariantRef left, VariantRef right) { - return equals_node(left, right, 0); + return canonical_compare(left, right) == 0; +} + +int canonical_compare(VariantRef left, VariantRef right) { + return compare_node(left, right, 0); } template @@ -1018,10 +1201,21 @@ template void canonical_hash(VariantRef value, VariantCrc3 template void canonical_hash(VariantRef value, VariantCrc32cHashSink& sink); bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right) { + return canonical_compare(left, right) == 0; +} + +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right) { const NormalizedValue normalized_left = VariantScalarAdapter::normalize(left); const NormalizedValue normalized_right = VariantScalarAdapter::normalize(right); - return normalized_left.kind == normalized_right.kind && - scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); +} + +int canonical_compare(const VariantScalarRef& left, VariantRef right) { + require_exact_value(right); + // A scalar never has a container kind, so an object or array on the right is decided by the + // kind order inside scalar_compare(), as compare_node() decides it, without walking the + // container. + return scalar_compare(VariantScalarAdapter::normalize(left), normalize_value(right)); } template diff --git a/be/src/core/value/variant/variant_canonical.h b/be/src/core/value/variant/variant_canonical.h index 286b43f94ec723..15d78e1334613d 100644 --- a/be/src/core/value/variant/variant_canonical.h +++ b/be/src/core/value/variant/variant_canonical.h @@ -35,6 +35,11 @@ struct VariantScalarAdapter; bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right); +// Returns a negative value, zero, or a positive value when left is respectively less than, equal +// to, or greater than right in the canonical Variant total order. Zero is exactly equivalent to +// canonical_equals(). +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right); + template void canonical_hash(const VariantScalarRef& value, Sink& sink); @@ -77,8 +82,28 @@ class VariantCrc32cHashSink { // Compares logical Variant values. Numeric normalization is deliberately limited to the exact // integer domain that canonical Variant decimal16 can encode: [-(10^38-1), +(10^38-1)]. A finite // integral float/double outside that domain remains floating so canonical arena bytes stay valid. +// +// The canonical equality contract for numbers: +// - an integer, a decimal with a zero fraction and an integral float/double in the domain above +// are one value: 1, 1.0 (decimal) and 1.0 (double) are equal; +// - a decimal and a float/double with a fraction are never equal, even when they print the same: +// DECIMAL 1.5 != DOUBLE 1.5. They order by numeric value, and at a numeric tie the decimal +// sorts first. Equating them would need a lossless common form, and the exact value of most +// doubles (0.1 for one) does not fit the 38 digits of a Variant decimal. +// canonical_equals(), canonical_compare() == 0, canonical_hash() and the canonical serialized key +// (GROUP BY, DISTINCT, join build) all implement this one rule, and sort peers follow it. A change +// to the rule has to change all of them together. bool canonical_equals(VariantRef left, VariantRef right); +// Container values are compared lexicographically after scalar normalization. Object keys use +// their canonical byte order. Numeric kinds share a numeric order, with canonical kind as a +// deterministic tie-breaker when two numerically equal values are not canonically equal. +int canonical_compare(VariantRef left, VariantRef right); + +// Compares a typed scalar with an encoded root in the same order, without encoding the scalar: +// canonical_compare(left, right) == canonical_compare(encode(left), right). +int canonical_compare(const VariantScalarRef& left, VariantRef right); + // Hashes canonical logical tokens without re-encoding the value. Supported production sinks are // SipHash, VariantXxHashSink, VariantCrc32HashSink, and VariantCrc32cHashSink; all four are explicit // instantiations of the same traversal skeleton. diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index 4270dfea08bf03..817e09de98b6e1 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -18,6 +18,7 @@ #include "exec/common/variant_util.h" #include +#include #include #include #include @@ -35,9 +36,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -81,7 +80,6 @@ #include "exprs/function/simple_function_factory.h" #include "exprs/function_context.h" #include "exprs/json_functions.h" -#include "re2/re2.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "storage/olap_common.h" @@ -141,162 +139,6 @@ void append_empty_key_subcolumn_from_stats(VariantCompactionPaths& paths_set_inf } // namespace -inline void append_escaped_regex_char(std::string* regex_output, char ch) { - switch (ch) { - case '.': - case '^': - case '$': - case '+': - case '*': - case '?': - case '(': - case ')': - case '|': - case '{': - case '}': - case '[': - case ']': - case '\\': - regex_output->push_back('\\'); - regex_output->push_back(ch); - break; - default: - regex_output->push_back(ch); - break; - } -} - -// Small LRU to cap compiled glob patterns -constexpr size_t kGlobRegexCacheCapacity = 256; - -struct GlobRegexCacheEntry { - std::shared_ptr re2; - std::list::iterator lru_it; -}; - -static std::mutex g_glob_regex_cache_mutex; -static std::list g_glob_regex_cache_lru; -static std::unordered_map g_glob_regex_cache; - -std::shared_ptr get_or_build_re2(const std::string& glob_pattern) { - { - std::lock_guard lock(g_glob_regex_cache_mutex); - auto it = g_glob_regex_cache.find(glob_pattern); - if (it != g_glob_regex_cache.end()) { - g_glob_regex_cache_lru.splice(g_glob_regex_cache_lru.begin(), g_glob_regex_cache_lru, - it->second.lru_it); - return it->second.re2; - } - } - std::string regex_pattern; - Status st = glob_to_regex(glob_pattern, ®ex_pattern); - if (!st.ok()) { - return nullptr; - } - auto compiled = std::make_shared(regex_pattern); - if (!compiled->ok()) { - return nullptr; - } - { - std::lock_guard lock(g_glob_regex_cache_mutex); - auto it = g_glob_regex_cache.find(glob_pattern); - if (it != g_glob_regex_cache.end()) { - g_glob_regex_cache_lru.splice(g_glob_regex_cache_lru.begin(), g_glob_regex_cache_lru, - it->second.lru_it); - return it->second.re2; - } - g_glob_regex_cache_lru.push_front(glob_pattern); - g_glob_regex_cache.emplace(glob_pattern, - GlobRegexCacheEntry {compiled, g_glob_regex_cache_lru.begin()}); - if (g_glob_regex_cache.size() > kGlobRegexCacheCapacity) { - const std::string& evict_key = g_glob_regex_cache_lru.back(); - g_glob_regex_cache.erase(evict_key); - g_glob_regex_cache_lru.pop_back(); - } - } - return compiled; -} - -// Convert a restricted glob pattern into a regex. -// Supported: '*', '?', '[...]', '\\' escape. Others are treated as literals. -Status glob_to_regex(const std::string& glob_pattern, std::string* regex_pattern) { - regex_pattern->clear(); - regex_pattern->append("^"); - bool is_escaped = false; - size_t pattern_length = glob_pattern.size(); - for (size_t index = 0; index < pattern_length; ++index) { - char current_char = glob_pattern[index]; - if (is_escaped) { - append_escaped_regex_char(regex_pattern, current_char); - is_escaped = false; - continue; - } - if (current_char == '\\') { - is_escaped = true; - continue; - } - if (current_char == '*') { - regex_pattern->append(".*"); - continue; - } - if (current_char == '?') { - regex_pattern->append("."); - continue; - } - if (current_char == '[') { - size_t class_index = index + 1; - bool class_closed = false; - bool is_class_escaped = false; - std::string class_buffer; - if (class_index < pattern_length && - (glob_pattern[class_index] == '!' || glob_pattern[class_index] == '^')) { - class_buffer.push_back('^'); - ++class_index; - } - for (; class_index < pattern_length; ++class_index) { - char class_char = glob_pattern[class_index]; - if (is_class_escaped) { - class_buffer.push_back(class_char); - is_class_escaped = false; - continue; - } - if (class_char == '\\') { - is_class_escaped = true; - continue; - } - if (class_char == ']') { - class_closed = true; - break; - } - class_buffer.push_back(class_char); - } - if (!class_closed) { - return Status::InvalidArgument("Unclosed character class in glob pattern: {}", - glob_pattern); - } - regex_pattern->append("["); - regex_pattern->append(class_buffer); - regex_pattern->append("]"); - index = class_index; - continue; - } - append_escaped_regex_char(regex_pattern, current_char); - } - if (is_escaped) { - append_escaped_regex_char(regex_pattern, '\\'); - } - regex_pattern->append("$"); - return Status::OK(); -} - -bool glob_match_re2(const std::string& glob_pattern, const std::string& candidate_path) { - auto compiled = get_or_build_re2(glob_pattern); - if (compiled == nullptr) { - return false; - } - return RE2::FullMatch(candidate_path, *compiled); -} - // NestedGroup's physical children and offsets are produced by NestedGroupWriteProvider, not by // appending TabletSchema extracted columns here. This predicate keeps only ordinary Variant paths // that are outside the NG tree, for example `v.owner` beside `v.items[*]`. @@ -1633,7 +1475,8 @@ bool generate_sub_column_info(const TabletSchema& schema, int32_t col_unique_id, break; } case PatternTypePB::MATCH_NAME_GLOB: { - if (glob_match_re2(pattern, path)) { + int result = fnmatch(pattern, path.c_str(), FNM_PATHNAME); + if (result == 0) { generate_result_column(*sub_column, &sub_column_info->column); generate_index(sub_column->name()); return true; diff --git a/be/src/exec/common/variant_util.h b/be/src/exec/common/variant_util.h index 5c8c1516192545..2cfc7f71a0c7ea 100644 --- a/be/src/exec/common/variant_util.h +++ b/be/src/exec/common/variant_util.h @@ -59,12 +59,6 @@ const std::string SPARSE_COLUMN_PATH = "__DORIS_VARIANT_SPARSE__"; const std::string DOC_VALUE_COLUMN_PATH = "__DORIS_VARIANT_DOC_VALUE__"; namespace doris::variant_util { -// Convert a restricted glob pattern into a regex (for tests/internal use). -Status glob_to_regex(const std::string& glob_pattern, std::string* regex_pattern); - -// Match a glob pattern against a path using RE2. -bool glob_match_re2(const std::string& glob_pattern, const std::string& candidate_path); - using PathToNoneNullValues = std::unordered_map; using PathToDataTypes = std::unordered_map, PathInData::Hash>; diff --git a/be/src/exec/sort/sort_block.cpp b/be/src/exec/sort/sort_block.cpp index e9f033501f682e..bb7d67ff03fb32 100644 --- a/be/src/exec/sort/sort_block.cpp +++ b/be/src/exec/sort/sort_block.cpp @@ -21,8 +21,15 @@ #include "exec/sort/sort_block.h" #include "core/block/column_with_type_and_name.h" +#include "core/column/variant_v2/column_variant_v2.h" namespace doris { +void ColumnSorter::sort_column(const ColumnVariantV2& column, EqualFlags& flags, + IColumn::Permutation& perms, EqualRange& range, + bool last_column) const { + _sort_by_default(column, flags, perms, range, last_column); +} + ColumnsWithSortDescriptions get_columns_with_sort_description(const Block& block, const SortDescription& description) { size_t size = description.size(); diff --git a/be/src/exec/sort/sort_block.h b/be/src/exec/sort/sort_block.h index 8d051bd28e9b12..a59b68e8e61294 100644 --- a/be/src/exec/sort/sort_block.h +++ b/be/src/exec/sort/sort_block.h @@ -51,6 +51,7 @@ template class ColumnDecimal; template class ColumnVector; +class ColumnVariantV2; } // namespace doris namespace doris { @@ -252,6 +253,8 @@ class ColumnSorter { EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); } + void sort_column(const ColumnVariantV2& column, EqualFlags& flags, IColumn::Permutation& perms, + EqualRange& range, bool last_column) const; void sort_column(const ColumnVarbinary& column, EqualFlags& flags, IColumn::Permutation& perms, EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); @@ -387,7 +390,8 @@ class ColumnSorter { !std::is_same_v && !std::is_same_v && !std::is_same_v && - !std::is_same_v) { + !std::is_same_v && + !std::is_same_v) { auto value_a = column.get_data()[a]; auto value_b = column.get_data()[b]; return Compare::compare(value_a, value_b); diff --git a/be/src/exprs/function/cast/cast_to_jsonb.h b/be/src/exprs/function/cast/cast_to_jsonb.h index 60393ef622550d..b263803583202b 100644 --- a/be/src/exprs/function/cast/cast_to_jsonb.h +++ b/be/src/exprs/function/cast/cast_to_jsonb.h @@ -24,6 +24,7 @@ #include "core/value/jsonb_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_string.h" +#include "runtime/runtime_state.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -240,8 +241,10 @@ WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const Dat auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, - null_map)); + options, null_map)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); }; diff --git a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp index c6d400ca314899..ca1e858caadf47 100644 --- a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp +++ b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp @@ -502,7 +502,12 @@ Status cast_scalar_to_variant(const ColumnPtr& source, const DataTypePtr& source } ColumnPtr null_map = std::move(nulls); ColumnPtr nullable = ColumnNullable::create(source, null_map); - *output = ColumnVariantV2::create_typed(std::move(nullable), source_type); + auto typed = ColumnVariantV2::create_typed(std::move(nullable), source_type); + // A scalar column can hold values Variant cannot, such as a string that is not UTF-8. Reject + // them here, as encoding would, so the typed state never compares or hashes a value that its + // encoded form could not hold. The CAST wrapper turns the exception into the statement error. + typed->validate_typed_rows(); + *output = std::move(typed); return Status::OK(); } diff --git a/be/src/exprs/function/function_to_json.cpp b/be/src/exprs/function/function_to_json.cpp index 9d84f3551d0922..5798d942e70f47 100644 --- a/be/src/exprs/function/function_to_json.cpp +++ b/be/src/exprs/function/function_to_json.cpp @@ -16,7 +16,10 @@ // under the License. #include "core/data_type/data_type_jsonb.h" +#include "core/data_type_serde/data_type_serde.h" #include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "runtime/runtime_state.h" namespace doris { @@ -42,8 +45,10 @@ class FunctionToJson : public IFunction { auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; - RETURN_IF_ERROR( - from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column)); + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); + RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, + options)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); } diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index 5523d6c2f9b99e..5f6f6990f33496 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -929,12 +929,12 @@ void append_value(VariantRef value, const DataTypePtr& target_type, IColumn* tar Status stringify_complex_column(const DataTypePtr& source_type, const ColumnNullable& source, ColumnPtr* result) { + DataTypeSerDe::FormatOptions options; auto jsonb = ColumnString::create(); RETURN_IF_ERROR(source_type->get_serde()->serialize_column_to_jsonb_vector( - source.get_nested_column(), *jsonb)); + source.get_nested_column(), *jsonb, options)); auto strings = ColumnString::create(); - DataTypeSerDe::FormatOptions options; DataTypeJsonb().get_serde()->to_string_batch(*jsonb, *strings, options); *result = ColumnNullable::create(std::move(strings), source.get_null_map_column().clone_resized(source.size())); diff --git a/be/test/core/column/column_variant_v2_test.cpp b/be/test/core/column/column_variant_v2_test.cpp index bde8e8f3e5af97..90eae92e41c016 100644 --- a/be/test/core/column/column_variant_v2_test.cpp +++ b/be/test/core/column/column_variant_v2_test.cpp @@ -66,6 +66,7 @@ #include "exec/common/hash_table/string_hash_map.h" #include "exec/common/sip_hash.h" #include "exec/sort/hybrid_sorter.h" +#include "exec/sort/sort_block.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" #include "runtime/memory/mem_tracker.h" @@ -288,6 +289,50 @@ class CountingShreddedState final : public VariantShreddedState { ColumnVariantV2::MutablePtr _serialized; }; +// A complete shredded state over known rows that counts how often its canonical rows are asked for. +class MaterializingShreddedState final : public VariantShreddedState { +public: + MaterializingShreddedState(ColumnPtr rows, std::shared_ptr materialized_calls) + : _rows(std::move(rows)), _materialized_calls(std::move(materialized_calls)) {} + + size_t size() const override { return _rows->size(); } + size_t byte_size() const override { return 0; } + size_t allocated_bytes() const override { return 0; } + void sanity_check() const override {} + void for_each_subcolumn(IColumn::ColumnCallback) const override {} + std::shared_ptr filter(const IColumn::Filter&, ssize_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_range(size_t, size_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_indices(const uint32_t*, + const uint32_t*) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + bool can_materialize() const override { return true; } + bool try_append(const VariantShreddedState&) override { return false; } + std::optional find_typed_value( + std::span) const override { + return std::nullopt; + } + std::optional find_normalized_value( + std::span) const override { + return std::nullopt; + } + const ColumnVariantV2& materialized_column() const override { + ++*_materialized_calls; + return assert_cast(*_rows); + } + const ColumnVariantV2& serialized_column() const override { + return assert_cast(*_rows); + } + +private: + ColumnPtr _rows; + std::shared_ptr _materialized_calls; +}; + template void expect_not_implemented(Function&& function, std::string_view marker) { try { @@ -468,6 +513,20 @@ void validate_encoded_column(const ColumnVariantV2& column) { } } +// The encoded twin of a typed column. insert_range_from() into an empty column keeps the source's +// typed state, so a test that wants the other representation has to encode explicitly; both +// physical states are asserted so a variable name cannot stand in for coverage. +ColumnVariantV2::MutablePtr encoded_copy_of(const ColumnVariantV2& typed) { + EXPECT_TRUE(typed.is_typed()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(typed, 0, typed.size()); + encoded->ensure_encoded(); + EXPECT_FALSE(encoded->is_typed()); + EXPECT_FALSE(encoded->is_shredded()); + EXPECT_EQ(encoded->size(), typed.size()); + return encoded; +} + void ensure_typed_fields_match_direct_encoding(ColumnVariantV2& column) { ASSERT_TRUE(column.is_typed()); std::vector fields(column.size()); @@ -1523,6 +1582,47 @@ TEST(ColumnVariantV2Test, CompositeShreddedSizeDoesNotRecountSegments) { EXPECT_EQ(*size_calls, 0); } +TEST(ColumnVariantV2Test, EraseRemovesRowsInEveryState) { + // Encoded rows with two metadata dictionaries: the ids of the remaining rows stay valid. + auto encoded = ColumnVariantV2::create(); + for (const char* json : {R"({"a":1})", "2", R"({"b":3})", "4", R"("five")"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + encoded->erase(0, 2); + ASSERT_EQ(encoded->size(), 3); + EXPECT_EQ(json_at(*encoded, 0), R"({"b":3})"); + EXPECT_EQ(json_at(*encoded, 2), R"("five")"); + encoded->erase(1, 1); + ASSERT_EQ(encoded->size(), 2); + EXPECT_EQ(json_at(*encoded, 1), R"("five")"); + encoded->erase(2, 0); + EXPECT_EQ(encoded->size(), 2); + validate_encoded_column(*encoded); + + constexpr std::array VALUES {7, 8, 9, 10}; + constexpr std::array NULLS {0, 1, 0, 0}; + auto typed = typed_int32(VALUES, NULLS); + typed->erase(0, 2); + EXPECT_TRUE(typed->is_typed()); + constexpr std::array REMAINING {9, 10}; + constexpr std::array REMAINING_NULLS {0, 0}; + expect_int32_rows(*typed, REMAINING, REMAINING_NULLS); + + auto rows = ColumnVariantV2::create(); + for (const char* json : {"1", "2", "3"}) { + insert_encoded_field(*rows, encode_json(json)); + } + auto calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(rows->get_ptr(), calls)); + shredded->erase(0, 1); + EXPECT_FALSE(shredded->is_shredded()); + ASSERT_EQ(shredded->size(), 2); + EXPECT_EQ(json_at(*shredded, 0), "2"); + EXPECT_EQ(json_at(*shredded, 1), "3"); + EXPECT_EQ(rows->size(), 3); +} + TEST(ColumnVariantV2Test, PopBackAndResizeCoverBoundsShrinkAndGrowth) { auto pop_column = ColumnVariantV2::create(); auto pop_reference = ColumnString::create(); @@ -2430,6 +2530,30 @@ TEST(ColumnVariantV2Test, TypedEnsureEncodedAllScalarMappings) { ASSERT_EQ(zero_dates.size(), 2); EXPECT_EQ(zero_dates.get_data()[0], valid_date); EXPECT_EQ(zero_dates.get_data()[1], invalid_date); + + // validate_typed_rows() is the check CAST runs on arbitrary scalars. It reads the rows the + // typed null map leaves visible and nothing else. + EXPECT_THROW(zero_date->validate_typed_rows(), Exception); + auto masked_nested = ColumnDateV2::create(); + masked_nested->insert_value(valid_date); + masked_nested->insert_value(invalid_date); + auto masked_nulls = ColumnUInt8::create(); + masked_nulls->insert_value(0); + masked_nulls->insert_value(1); + auto masked = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(masked_nested), std::move(masked_nulls)), + std::make_shared()); + EXPECT_NO_THROW(masked->validate_typed_rows()); + masked->ensure_encoded(); + validate_encoded_column(*masked); + EXPECT_TRUE(masked->get_value_ref(1).is_null()); + + constexpr std::array INVALID_UTF8_ROWS {"ok", "\xFF"}; + constexpr std::array INVALID_UTF8_VISIBLE {0, 0}; + constexpr std::array INVALID_UTF8_HIDDEN {0, 1}; + EXPECT_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_VISIBLE)->validate_typed_rows(), + Exception); + EXPECT_NO_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_HIDDEN)->validate_typed_rows()); } TEST(ColumnVariantV2Test, TypedRowTransformsCloneAndCow) { @@ -2567,6 +2691,8 @@ TEST(ColumnVariantV2Test, TypedMutatorsDetachSharedNestedColumn) { constexpr std::array FIRST_TWO_NULLS {}; exercise([](ColumnVariantV2& column) { column.pop_back(1); }, FIRST_TWO, FIRST_TWO_NULLS); exercise([](ColumnVariantV2& column) { column.resize(2); }, FIRST_TWO, FIRST_TWO_NULLS); + constexpr std::array LAST_TWO {2, 3}; + exercise([](ColumnVariantV2& column) { column.erase(0, 1); }, LAST_TWO, FIRST_TWO_NULLS); constexpr std::array FILTERED {1, 3}; constexpr std::array FILTERED_NULLS {}; exercise( @@ -2836,6 +2962,41 @@ TEST(ColumnVariantV2Test, MixedEncodedTypedInsertAndGatherPreserveCanonicalRows) expect_canonical_rows_equal(*typed, *encoded); } +TEST(ColumnVariantV2Test, NullableBatchSerializationMatchesRowSerialization) { + constexpr std::array VALUES {42, 0, -7}; + constexpr std::array NULLS {0, 1, 0}; + auto typed = typed_int32(VALUES, NULLS); + auto encoded = ColumnVariantV2::create(); + insert_encoded_field(*encoded, encoded_integer(42, 4)); + insert_encoded_field(*encoded, encode_json("null")); + insert_encoded_field(*encoded, encoded_integer(-7, 4)); + for (const auto* column : {typed.get(), encoded.get()}) { + for (const bool has_null : {false, true}) { + std::array buffers; + std::array keys; + for (size_t row = 0; row < 3; ++row) { + buffers[row].assign(8 + column->get_max_row_byte_size(), '\0'); + buffers[row][0] = 'x'; + keys[row] = {buffers[row].data(), 1}; + } + constexpr std::array NO_NULLS {0, 0, 0}; + column->serialize_with_nullable(keys.data(), 3, has_null, + has_null ? NULLS.data() : NO_NULLS.data()); + for (size_t row = 0; row < 3; ++row) { + std::string expected = "x"; + const bool is_null = has_null && NULLS[row]; + expected.push_back(is_null ? 1 : 0); + if (!is_null) { + std::string cell(column->serialize_size_at(row), '\0'); + column->serialize_impl(cell.data(), row); + expected += cell; + } + EXPECT_EQ(std::string(keys[row].data, keys[row].size), expected); + } + } + } +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive E/T adapter matrix. TEST(ColumnVariantV2Test, TypedUuidMatchesNativeEncodingAndNulls) { const std::array bytes {0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, @@ -3231,21 +3392,309 @@ TEST(ColumnVariantV2Test, ETCrossCheckTemporalClassMatrix) { ntz_representations[2], 0); } -TEST(ColumnVariantV2Test, TypedPhysicalAndOrderingInterfacesStayUnsupported) { - constexpr std::array VALUES {1}; - constexpr std::array NULLS {0}; +TEST(ColumnVariantV2Test, TypedComparisonBoundariesMatchEncodedWithoutMaterializing) { + auto check = [](const ColumnVariantV2::MutablePtr& typed) { + ASSERT_TRUE(typed->is_typed()); + const auto* original = &typed->typed_column(); + auto encoded = encoded_copy_of(*typed); + ASSERT_FALSE(encoded->is_typed()); + for (size_t right = 0; right < typed->size(); ++right) { + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + const int expected = encoded->compare_at(left, right, *encoded, hint); + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), expected); + EXPECT_EQ(typed->compare_at(left, right, *encoded, hint), expected); + EXPECT_EQ(encoded->compare_at(left, right, *typed, hint), expected); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(right, *typed, hint, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(right, *encoded, hint, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + } + // Prove the tested input did not silently become encoded during comparison. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + check(ColumnVariantV2::create_typed( + nullable_fixed({0, -1, 9007199254740993LL, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-std::numeric_limits::infinity(), -0.0, 0.0, 1.5, + std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + std::bit_cast(uint64_t {0xfff8000000000001ULL}), 0.0}, + {0, 0, 0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 2, {Decimal64 {-150}, Decimal64 {0}, Decimal64 {150}, Decimal64 {0}}, + {0, 0, 0, 1}), + std::make_shared(18, 2))); + const auto date = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 2, 29)); + const auto next = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 3, 1)); + check(ColumnVariantV2::create_typed(nullable_fixed>( + {date, next, date}, {0, 0, 1}), + std::make_shared())); + // Address order differs from textual Variant order for these values. + IPv4 ip4_a {}, ip4_b {}; + ASSERT_TRUE(IPv4Value::from_string(ip4_a, "10.0.0.2")); + ASSERT_TRUE(IPv4Value::from_string(ip4_b, "10.0.0.10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip4_a, ip4_b, ip4_a}, {0, 0, 1}), + std::make_shared())); + IPv6 ip6_a {}, ip6_b {}; + ASSERT_TRUE(IPv6Value::from_string(ip6_a, "2001:db8::2")); + ASSERT_TRUE(IPv6Value::from_string(ip6_b, "2001:db8::10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip6_a, ip6_b, ip6_a}, {0, 0, 1}), + std::make_shared())); + // LARGEINT magnitudes above 10^38 - 1 are represented as Variant strings. + const __int128 limit = power_of_ten_i128(38); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-limit, Int128 {0}, limit - 1, limit, -(limit - 1), Int128 {0}}, + {0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_fixed({1, 0, 1}, {0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed({-1, 0, 127, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-1.25F, 0.5F, std::numeric_limits::quiet_NaN(), 0.0F}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 4, + {Decimal128V3 {Int128 {-15000}}, Decimal128V3 {Int128 {0}}, + Decimal128V3 {Int128 {15000}}, Decimal128V3 {Int128 {0}}}, + {0, 0, 0, 1}), + std::make_shared(38, 4))); + const auto datetime_a = + DateV2Value::create_from_olap_datetime(19700101000001ULL); + const auto datetime_b = + DateV2Value::create_from_olap_datetime(20240229123456ULL); + check(ColumnVariantV2::create_typed( + nullable_fixed>( + {datetime_b, datetime_a, datetime_b}, {0, 0, 1}), + std::make_shared(6))); + // Bytes at or above 0x80 pin the unsigned string order. + constexpr std::array STRINGS {"a", "\xC3\xA9", "z", ""}; + constexpr std::array STRING_NULLS {0, 0, 0, 1}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, TypedComparisonAcrossTypeParametersMatchesEncoded) { + // DECIMAL(18,2) and DECIMAL(18,3) share a column type but not a scale. + auto left = ColumnVariantV2::create_typed(nullable_decimal( + 2, {Decimal64 {150}, Decimal64 {-5}}, {0, 0}), + std::make_shared(18, 2)); + auto right = + ColumnVariantV2::create_typed(nullable_decimal( + 3, {Decimal64 {1400}, Decimal64 {-50}}, {0, 0}), + std::make_shared(18, 3)); + auto left_encoded = encoded_copy_of(*left); + auto right_encoded = encoded_copy_of(*right); + for (size_t right_row = 0; right_row < right->size(); ++right_row) { + for (size_t left_row = 0; left_row < left->size(); ++left_row) { + EXPECT_EQ(left->compare_at(left_row, right_row, *right, 1), + left_encoded->compare_at(left_row, right_row, *right_encoded, 1)); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(left->size(), 0); + std::vector encoded_cmp(left->size(), 0); + IColumn::Filter typed_filter(left->size(), 0); + IColumn::Filter encoded_filter(left->size(), 0); + left->compare_internal(right_row, *right, 1, direction, typed_cmp, typed_filter.data()); + left_encoded->compare_internal(right_row, *right_encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } +} + +TEST(ColumnVariantV2Test, TypedAgainstEncodedRootsMatchesEncoded) { + // The encoded side holds every kind of root a typed scalar can meet, including containers and + // canonical peers of the typed values (2 and 2.0, 1.5 as a decimal). + auto encoded_roots = ColumnVariantV2::create(); + for (const char* json : + {"null", "true", "1", "2", "2.0", "1.5", R"("a")", R"("2")", "[2]", R"({"k":2})"}) { + insert_encoded_field(*encoded_roots, encode_json(json)); + } + ASSERT_FALSE(encoded_roots->is_typed()); + + auto check = [&](const ColumnVariantV2::MutablePtr& typed) { + auto encoded = encoded_copy_of(*typed); + const auto* original = &typed->typed_column(); + for (size_t row = 0; row < typed->size(); ++row) { + for (size_t root = 0; root < encoded_roots->size(); ++root) { + const int expected = encoded->compare_at(row, root, *encoded_roots, 1); + EXPECT_EQ(typed->compare_at(row, root, *encoded_roots, 1), expected) + << "row=" << row << " root=" << root; + EXPECT_EQ(encoded_roots->compare_at(root, row, *typed, 1), -expected) + << "row=" << row << " root=" << root; + } + } + for (size_t root = 0; root < encoded_roots->size(); ++root) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(root, *encoded_roots, 1, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(root, *encoded_roots, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + // Comparing must not change the state of the typed column. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + constexpr std::array INTEGERS {1, 2, 3, 0}; + constexpr std::array INTEGER_NULLS {0, 0, 0, 1}; + check(typed_int32(INTEGERS, INTEGER_NULLS)); + check(ColumnVariantV2::create_typed( + nullable_fixed({1.5, 2.0, std::numeric_limits::quiet_NaN()}, + {0, 0, 0}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_decimal( + 1, {Decimal64 {15}, Decimal64 {20}}, {0, 0}), + std::make_shared(18, 1))); + constexpr std::array STRINGS {"a", "2", "b"}; + constexpr std::array STRING_NULLS {0, 0, 0}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, ShreddedComparisonMatchesEncodedAndStaysShredded) { + auto encoded = ColumnVariantV2::create(); + for (const char* json : {"2", "1.5", R"("a")", R"({"k":1})", "null"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + auto materialized_calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(encoded->get_ptr(), materialized_calls)); + ASSERT_TRUE(shredded->is_shredded()); + constexpr std::array INTEGERS {1, 2, 3}; + constexpr std::array NOT_NULL {0, 0, 0}; + auto typed = typed_int32(INTEGERS, NOT_NULL); + auto typed_encoded = encoded_copy_of(*typed); + + for (size_t left = 0; left < encoded->size(); ++left) { + for (size_t right = 0; right < encoded->size(); ++right) { + const int expected = encoded->compare_at(left, right, *encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *shredded, 1), expected); + EXPECT_EQ(shredded->compare_at(left, right, *encoded, 1), expected); + EXPECT_EQ(encoded->compare_at(left, right, *shredded, 1), expected); + } + for (size_t right = 0; right < typed->size(); ++right) { + const int expected = encoded->compare_at(left, right, *typed_encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *typed, 1), expected); + EXPECT_EQ(typed->compare_at(right, left, *shredded, 1), -expected); + } + } + // Comparison reads the rows the state materializes (a real state caches them); the column + // itself stays shredded, and the typed peer stays typed. + EXPECT_GT(*materialized_calls, 0); + EXPECT_TRUE(shredded->is_shredded()); + EXPECT_TRUE(typed->is_typed()); +} + +TEST(ColumnVariantV2Test, TypedNaNOrderingIgnoresNullDirectionHint) { + auto typed = ColumnVariantV2::create_typed( + nullable_fixed({1.0, std::numeric_limits::quiet_NaN()}, {0, 0}), + std::make_shared()); + auto encoded = encoded_copy_of(*typed); + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), + encoded->compare_at(left, right, *encoded, hint)); + } + } + } +} + +TEST(ColumnVariantV2Test, TypedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { + constexpr std::array VALUES {2, 99, 1, 3}; + constexpr std::array NULLS {0, 1, 0, 0}; auto typed = typed_int32(VALUES, NULLS); expect_not_implemented([&] { static_cast(typed->get_data_at(0)); }, "intentionally unsupported"); + auto encoded = encoded_copy_of(*typed); + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + for (int nan_direction_hint : {-1, 1}) { + EXPECT_EQ(typed->compare_at(left, right, *typed, nan_direction_hint), + encoded->compare_at(left, right, *encoded, nan_direction_hint)); + } + } + } + for (size_t right = 0; right < typed->size(); ++right) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(encoded->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(encoded->size(), 0); + typed->compare_internal(right, *typed, 1, direction, typed_cmp, typed_filter.data()); + encoded->compare_internal(right, *encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { typed->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); + typed->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {1, 2, 0, 3})); + typed->get_permutation(true, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {3, 0})); expect_not_implemented([&] { typed->replace_column_data(*typed, 0); }, "intentionally unsupported"); EXPECT_TRUE(typed->is_typed()); } +TEST(ColumnVariantV2Test, PermutationLimitSwitchesBetweenPartialAndFullSort) { + // get_permutation only uses std::partial_sort while the limit stays small relative to the row + // count, the threshold ColumnVector applies as well; a larger limit falls back to the full + // sort. Both branches must order the leading rows the same way. + constexpr std::array VALUES {9, 3, 14, 7, 1, 12, 5, 15, 2, 11, 6, 13, 4, 10, 8, 0}; + constexpr std::array NULLS {}; + auto typed = typed_int32(VALUES, NULLS); + + HybridSorter sorter; + IColumn::Permutation result; + // limit 2 of 16 rows sits exactly on the threshold and still takes the partial_sort branch. + // Pin the whole prefix that branch must produce rather than only its first entry: the values 0 + // and 1 live at rows 15 and 4. The two branches cannot be told apart through the public API, so + // this pins the contract both of them owe, not the branch itself. + typed->get_permutation(false, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {15, 4})); + + // limit 12 of 16 rows exceeds the threshold, so the whole permutation comes out ordered. + typed->get_permutation(false, 12, 0, sorter, result); + ASSERT_EQ(result.size(), VALUES.size()); + for (size_t index = 1; index < result.size(); ++index) { + EXPECT_LT(typed->compare_at(result[index - 1], result[index], *typed, 1), 0); + } +} + TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { auto encoded = ColumnVariantV2::create(); insert_encoded_field(*encoded, encode_json("1")); @@ -3268,19 +3717,90 @@ TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { EXPECT_EQ(json_at(*typed, 2), "{}"); } -TEST(ColumnVariantV2Test, EncodedPhysicalAndOrderingInterfacesStayUnsupported) { +TEST(ColumnVariantV2Test, EncodedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { auto column = ColumnVariantV2::create(); - auto source = ColumnVariantV2::create(); - insert_encoded_field(*source, encode_json("1")); + insert_encoded_field(*column, encode_json(R"("text")")); + insert_encoded_field(*column, encode_json("2")); + insert_encoded_field(*column, encode_json("1.5")); + insert_encoded_field(*column, encode_json("true")); expect_not_implemented([&] { static_cast(column->get_data_at(0)); }, "intentionally unsupported"); HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { column->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); - expect_not_implemented([&] { column->replace_column_data(*source, 0); }, + column->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {3, 2, 1, 0})); + expect_not_implemented([&] { column->replace_column_data(*column, 0); }, "intentionally unsupported"); } +TEST(ColumnVariantV2Test, NullableMultiKeyOrderingWorks) { + auto variant = ColumnVariantV2::create(); + insert_encoded_field(*variant, encode_json("2")); + insert_encoded_field(*variant, encode_json("1")); + insert_encoded_field(*variant, encode_json("1")); + auto nullable = ColumnNullable::create(std::move(variant), ColumnUInt8::create(3, 0)); + + auto tie = ColumnInt32::create(); + tie->insert_value(0); + tie->insert_value(2); + tie->insert_value(1); + auto ids = ColumnInt32::create(); + ids->insert_value(1); + ids->insert_value(2); + ids->insert_value(3); + + Block source; + source.insert({std::move(nullable), make_nullable(std::make_shared()), "v"}); + source.insert({std::move(tie), std::make_shared(), "tie"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, 1, 1); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + + const auto& sorted_ids = assert_cast(*sorted.get_by_position(2).column); + EXPECT_EQ(sorted_ids.get_data()[0], 3); + EXPECT_EQ(sorted_ids.get_data()[1], 2); + EXPECT_EQ(sorted_ids.get_data()[2], 1); +} + +TEST(ColumnVariantV2Test, NullableTypedSortMatchesEncoded) { + constexpr std::array VALUES {3, 0, 1, 7, 1}; + // Row 1 is a Variant null inside the value; row 2 is a SQL NULL in the outer column. + constexpr std::array VARIANT_NULLS {0, 1, 0, 0, 0}; + constexpr std::array SQL_NULLS {0, 0, 1, 0, 0}; + auto sorted_ids = [&](const ColumnPtr& variant, int direction, int nulls_direction) { + auto nulls = ColumnUInt8::create(); + nulls->get_data().assign(SQL_NULLS.begin(), SQL_NULLS.end()); + auto ids = ColumnInt32::create(); + for (int32_t id = 0; id < static_cast(VALUES.size()); ++id) { + ids->insert_value(id); + } + Block source; + source.insert({ColumnNullable::create(variant, std::move(nulls)), + make_nullable(std::make_shared()), "v"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, direction, nulls_direction); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + const auto& output = assert_cast(*sorted.get_by_position(1).column); + return std::vector(output.get_data().begin(), output.get_data().end()); + }; + auto typed = typed_int32(VALUES, VARIANT_NULLS); + auto encoded = encoded_copy_of(*typed); + for (int direction : {-1, 1}) { + for (int nulls_direction : {-1, 1}) { + EXPECT_EQ(sorted_ids(typed->get_ptr(), direction, nulls_direction), + sorted_ids(encoded->get_ptr(), direction, nulls_direction)) + << "direction=" << direction << " nulls_direction=" << nulls_direction; + } + } +} + } // namespace doris diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp index 3f77440d48bcd7..c0e30f314ae7aa 100644 --- a/be/test/core/data_type/data_type_timestamp_ns_test.cpp +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -600,7 +600,9 @@ TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) { EXPECT_EQ(const_json->get_data_at(0).to_string(), "1970-01-01 00:00:00.000000001"); JsonbWriter jsonb_writer; - ASSERT_TRUE(serde->serialize_column_to_jsonb(*source, 1, jsonb_writer).ok()); + ASSERT_TRUE(serde->serialize_column_to_jsonb(*source, 1, jsonb_writer, + DataTypeSerDe::FormatOptions {}) + .ok()); EXPECT_EQ(JsonbToJson::jsonb_to_json_string(jsonb_writer.getOutput()->getBuffer(), jsonb_writer.getOutput()->getSize()), "\"2024-02-29 12:34:56.123456789\""); @@ -613,7 +615,9 @@ TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) { assert_cast(*source).get_element(1)); auto jsonb_values = ColumnString::create(); - ASSERT_TRUE(serde->serialize_column_to_jsonb_vector(*source, *jsonb_values).ok()); + ASSERT_TRUE(serde->serialize_column_to_jsonb_vector(*source, *jsonb_values, + DataTypeSerDe::FormatOptions {}) + .ok()); auto jsonb_vector_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); ASSERT_TRUE(serde->deserialize_column_from_jsonb_vector(*jsonb_vector_result, *jsonb_values, cast_params) diff --git a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp index 1b5dddf8e9e925..5cecf9d9cefa00 100644 --- a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp @@ -15,27 +15,35 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include +#include #include #include #include #include +#include #include "core/arena.h" #include "core/assert_cast.h" +#include "core/column/column_array.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_number.h" +#include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_variant_v2_serde.h" #include "core/string_buffer.hpp" +#include "core/value/variant/variant_batch_builder.h" +#include "exprs/function/parse/variant_string_parse.h" #include "util/jsonb_document.h" +#include "util/jsonb_utils.h" #include "util/jsonb_writer.h" namespace doris { @@ -103,6 +111,33 @@ std::string json_at(const DataTypeVariantV2SerDe& serde, const IColumn& column, return output->get_data_at(0).to_string(); } +ColumnVariantV2::MutablePtr encoded_json(std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (std::string_view row : rows) { + encoder.add_json({row.data(), row.size()}); + } + VariantBatchBuilder block = encoder.finish_batch(); + auto result = ColumnVariantV2::create(); + result->insert_encoded_batch(block); + return result; +} + +std::vector jsonb_vector_as_json( + const DataTypeSerDe& serde, const IColumn& column, + const DataTypeSerDe::FormatOptions& options = DataTypeSerDe::FormatOptions {}) { + auto output = ColumnString::create(); + const Status status = serde.serialize_column_to_jsonb_vector(column, *output, options); + EXPECT_TRUE(status.ok()) << status; + std::vector rows; + for (size_t row = 0; row < output->size(); ++row) { + const StringRef bytes = output->get_data_at(row); + rows.push_back(JsonbToJson::jsonb_to_json_string(bytes.data, bytes.size)); + } + return rows; +} + std::string object_with_value(const std::function& write_value) { JsonbWriter writer; EXPECT_TRUE(writer.writeStartObject()); @@ -114,6 +149,51 @@ std::string object_with_value(const std::function& write_val } // namespace +TEST(DataTypeVariantV2SerdeJsonbTest, VectorSkipsMaskedTypedNonFiniteValues) { + auto values = ColumnFloat64::create(); + values->insert_value(1.5); + values->insert_value(std::numeric_limits::quiet_NaN()); + values->insert_value(2.5); + auto column = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(values), ColumnUInt8::create(3, 0)), + std::make_shared()); + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + auto output = ColumnString::create(); + const std::array null_map {0, 1, 0}; + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), "1.5"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), "2.5"); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, VectorPreservesMaskedEncodedAndConstantRowPositions) { + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + const std::array null_map {0, 1, 0}; + auto encoded = encoded_json({"1", "null", "2"}); + auto constant = ColumnConst::create(typed_int(42), 3); + for (const IColumn* column : {static_cast(encoded.get()), + static_cast(constant.get())}) { + auto output = ColumnString::create(); + ASSERT_TRUE( + serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), + column == encoded.get() ? "1" : "42"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), + column == encoded.get() ? "2" : "42"); + } +} + TEST(DataTypeVariantV2SerdeJsonbTest, EncodedAndTypedWriteIdenticalBinaryDocumentsAndRoundTrip) { DataTypeVariantV2SerDe serde; auto typed = typed_int(42); @@ -243,4 +323,80 @@ TEST(DataTypeVariantV2SerdeJsonbTest, InvalidRowDoesNotTouchResultWriter) { EXPECT_EQ(writer.getOutput()->getSize(), before); } +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbWritesTheJsonDocument) { + DataTypeVariantV2SerDe serde; + auto encoded = encoded_json({R"({"b":[true,null,"x"],"a":{"c":1.5}})", "42", R"("s")", "null"}); + EXPECT_EQ(jsonb_vector_as_json(serde, *encoded), + (std::vector {R"({"a":{"c":1.5},"b":[true,null,"x"]})", "42", R"("s")", + "null"})); + + // Typed and encoded columns write the same bytes, including a Variant null. + for (const bool is_null : {false, true}) { + auto typed = typed_int(42, is_null); + ColumnPtr encoded_int = encoded_copy(*typed); + auto typed_output = ColumnString::create(); + auto encoded_output = ColumnString::create(); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*typed, *typed_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*encoded_int, *encoded_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_EQ(typed_output->size(), 1); + EXPECT_EQ(typed_output->get_data_at(0), encoded_output->get_data_at(0)); + EXPECT_EQ(jsonb_vector_as_json(serde, *typed)[0], is_null ? "null" : "42"); + } + + ColumnPtr constant = ColumnConst::create(encoded_json({R"({"k":"v"})"})->get_ptr(), 2); + EXPECT_EQ(jsonb_vector_as_json(serde, *constant), + (std::vector {R"({"k":"v"})", R"({"k":"v"})"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbNestsVariantInArray) { + auto values = encoded_json({R"({"a":[1,2]})", "7", R"("x")"}); + auto value_nulls = ColumnUInt8::create(); + for (const uint8_t is_null : {0, 1, 0}) { + value_nulls->insert_value(is_null); + } + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(2); + offsets->insert_value(3); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), std::move(value_nulls)), std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array), + (std::vector {R"([{"a":[1,2]},null])", R"(["x"])"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbUsesTheSessionTimeZone) { + VariantBatchBuilder builder; + auto row = builder.begin_row(); + row.add_timestamp_micros(0, true); + row.finish(); + VariantBatchBuilder block = builder.finish_batch(); + auto values = ColumnVariantV2::create(); + values->insert_encoded_batch(block); + + DataTypeVariantV2SerDe serde; + const cctz::time_zone shanghai = cctz::fixed_time_zone(cctz::seconds(8 * 3600)); + DataTypeSerDe::FormatOptions options; + options.timezone = &shanghai; + EXPECT_EQ(jsonb_vector_as_json(serde, *values, options), + (std::vector {R"("1970-01-01 08:00:00.000000+08:00")"})); + EXPECT_EQ(jsonb_vector_as_json(serde, *values), + (std::vector {R"("1970-01-01 00:00:00.000000+00:00")"})); + + // A nested Variant element receives the same time zone. + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), ColumnUInt8::create(1, 0)), + std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array, options), + (std::vector {R"(["1970-01-01 08:00:00.000000+08:00"])"})); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp index 2972e7023da4ae..b8b1c598ae0261 100644 --- a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp @@ -53,7 +53,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt32::create(); @@ -65,7 +66,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt64::create(); EXPECT_TRUE(data.type->get_serde()->deserialize_column_from_jsonb( @@ -77,7 +79,8 @@ TEST(DataTypeWritToJsonb, test_number) { auto data = ColumnHelper::create_column_with_name( {1.1F, 2.2F, 3.3F, 4.4F, 5.5F}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4.4"); auto data_to = ColumnFloat32::create(); @@ -92,7 +95,8 @@ TEST(DataTypeWritToJsonb, test_number) { col->insert_value(val); auto type = std::make_shared(18, 2); JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer, + DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "123.45"); auto data_to = ColumnDecimal128V3::create(0, 2); @@ -109,7 +113,8 @@ TEST(DataTypeWritToJsonb, test_string) { auto data = ColumnHelper::create_column_with_name( {"hello", "world", "doris", "vectorized", "test"}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "\"vectorized\""); auto data_to = ColumnString::create(); @@ -137,7 +142,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[1,2,3]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -148,7 +154,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[null,5]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -182,7 +189,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":1,\"string_col\":\"hello\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -194,7 +202,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":2,\"string_col\":\"world\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -206,7 +215,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 2, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 2, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":3,\"string_col\":null}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -217,7 +227,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 3, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":null,\"string_col\":\"vectorized\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -229,7 +240,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 4, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 4, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":5,\"string_col\":\"test\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( diff --git a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp index 01a20d9e2da8ef..04526ee545ca42 100644 --- a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp +++ b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp @@ -95,6 +95,43 @@ TEST(CastVariantV2ToTest, UnsupportedSourceReturnsAnError) { EXPECT_EQ(cast.column.get(), cast.initial_result.get()); } +TEST(CastVariantV2ToTest, InvalidUtf8StringSourceReturnsAnError) { + const std::array source_types { + std::make_shared(), std::make_shared(16, TYPE_CHAR), + std::make_shared(64, TYPE_VARCHAR)}; + for (const DataTypePtr& source_type : source_types) { + auto source = ColumnString::create(); + source->insert_data("ok", 2); + source->insert_data("\xFF", 1); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) + << source_type->get_name() << ": " << cast.status; + + // A row that the cast turns into NULL never becomes a Variant string. + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << source_type->get_name() << ": " << null_row.status; + } +} + +TEST(CastVariantV2ToTest, InvalidDateSourceReturnsAnError) { + // The same check covers every scalar Variant cannot hold, not only strings. + auto source = ColumnDateV2::create(); + DateV2Value valid; + valid.unchecked_set_time(2024, 1, 2, 0, 0, 0); + DateV2Value invalid; + ASSERT_FALSE(invalid.is_valid_date()); + source->insert_value(valid); + source->insert_value(invalid); + const auto source_type = std::make_shared(); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) << cast.status; + + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << null_row.status; +} + TEST(CastVariantV2ToTest, Decimal256ReturnsAnErrorInsteadOfStringifying) { auto type = std::make_shared(76, 2); auto source = ColumnDecimal256::create(0, 2); diff --git a/be/test/exprs/variant_v2_execution_test.cpp b/be/test/exprs/variant_v2_execution_test.cpp index f87bbd282bdca2..e3db965fc222cb 100644 --- a/be/test/exprs/variant_v2_execution_test.cpp +++ b/be/test/exprs/variant_v2_execution_test.cpp @@ -18,8 +18,12 @@ #include #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" +#include "exprs/function/functions_comparison.h" +#include "exprs/function/parse/variant_string_parse.h" namespace doris { @@ -31,4 +35,79 @@ TEST(VariantV2ExecutionTest, ExecutionTypeSelectsPhysicalColumn) { EXPECT_NE(check_and_get_column(*compute_v2.create_column()), nullptr); } +TEST(VariantV2ExecutionTest, TypePropertiesDoNotAffectEquality) { + // Execution compares and hashes Variant values canonically; storage properties such as the + // maximum subcolumn count and doc mode must not make two Variant types different. + const DataTypePtr plain = std::make_shared(2048, false); + const DataTypePtr limited = std::make_shared(10, false); + const DataTypePtr doc_mode = std::make_shared(0, true); + EXPECT_TRUE(plain->equals(*limited)); + EXPECT_TRUE(limited->equals(*doc_mode)); + EXPECT_TRUE(make_nullable(plain)->equals(*make_nullable(doc_mode))); + EXPECT_FALSE(plain->equals(DataTypeInt32())); +} + +TEST(VariantV2ExecutionTest, CanonicalComparison) { + auto parse = [](std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 255, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (auto json : rows) { + encoder.add_json({json.data(), json.size()}); + } + auto column = ColumnVariantV2::create(); + column->insert_encoded_batch(encoder.finish_batch()); + return column; + }; + auto left = parse({"1", "1", "null", R"({"a":1,"b":[2,null]})", "[1,2]", "true"}); + auto right = parse({"1.0", R"("1")", "null", R"({"b":[2.0,null],"a":1.0})", "[2,1]", "1"}); + auto type = std::make_shared(); + Block block {{left->get_ptr(), type, "left"}, + {right->get_ptr(), type, "right"}, + {nullptr, std::make_shared(), "result"}}; + FunctionComparison equals; + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 0, 1, 1, 0, 0})); + FunctionComparison not_equals; + ASSERT_TRUE(not_equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + FunctionComparison less; + ASSERT_TRUE(less.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + + auto one = parse({"1.0"}); + block.get_by_position(1).column = ColumnConst::create(std::move(one), left->size()); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {1, 0}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); +} + +TEST(VariantV2ExecutionTest, TypedEquality) { + auto numbers = ColumnInt32::create(); + numbers->get_data() = {1, 2, 3}; + auto nulls = ColumnUInt8::create(); + nulls->get_data() = {0, 0, 1}; + auto typed = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(numbers), std::move(nulls)), + std::make_shared()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(*typed, 0, typed->size()); + encoded->ensure_encoded(); + ASSERT_TRUE(typed->is_typed()); + ASSERT_FALSE(encoded->is_typed()); + for (const auto& left : {typed->get_ptr(), encoded->get_ptr()}) { + for (const auto& right : {typed->get_ptr(), encoded->get_ptr()}) { + for (size_t row = 0; row < left->size(); ++row) { + EXPECT_EQ(left->compare_at(row, row, *right, 1), 0); + } + } + } +} + } // namespace doris diff --git a/be/test/storage/variant/variant_util_test.cpp b/be/test/storage/variant/variant_util_test.cpp deleted file mode 100644 index 0adbd43f285c06..00000000000000 --- a/be/test/storage/variant/variant_util_test.cpp +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "exec/common/variant_util.h" - -#include -#include - -#include "gtest/gtest.h" - -namespace doris::variant_util { - -TEST(VariantUtilTest, GlobToRegex) { - struct Case { - std::string glob; - std::string expected_regex; - }; - const std::vector cases = { - {"*", "^.*$"}, - {"?", "^.$"}, - {"a?b", "^a.b$"}, - {"a*b", "^a.*b$"}, - {"a**b", "^a.*.*b$"}, - {"a??b", "^a..b$"}, - {"?*", "^..*$"}, - {"*?", "^.*.$"}, - {"a.b", "^a\\.b$"}, - {"a+b", "^a\\+b$"}, - {"a{b}", "^a\\{b\\}$"}, - {R"(a\*b)", R"(^a\*b$)"}, - {"a\\?b", "^a\\?b$"}, - {"a\\[b", "^a\\[b$"}, - {"abc\\", "^abc\\\\$"}, - {"a|b", "^a\\|b$"}, - {"a(b)c", "^a\\(b\\)c$"}, - {"a^b", "^a\\^b$"}, - {"a$b", "^a\\$b$"}, - {"int_[0-9]", "^int_[0-9]$"}, - {"int_[!0-9]", "^int_[^0-9]$"}, - {"int_[^0-9]", "^int_[^0-9]$"}, - {"a[\\-]b", "^a[-]b$"}, - {"a[b-d]e", "^a[b-d]e$"}, - {"a[\\]]b", "^a[]]b$"}, - {"a[\\!]b", "^a[!]b$"}, - {"", "^$"}, - {"a[[]b", "^a[[]b$"}, - {"a[]b", "^a[]b$"}, - {"[]", "^[]$"}, - {"[!]", "^[^]$"}, - {"[^]", "^[^]$"}, - {"\\", "^\\\\$"}, - {"\\*", "^\\*$"}, - {"a\\*b", "^a\\*b$"}, - {"a[!\\]]b", "^a[^]]b$"}, - }; - - for (const auto& test_case : cases) { - std::string regex; - Status st = glob_to_regex(test_case.glob, ®ex); - EXPECT_TRUE(st.ok()) << st.to_string() << " pattern=" << test_case.glob; - EXPECT_EQ(regex, test_case.expected_regex) << "pattern=" << test_case.glob; - } - - std::string regex; - EXPECT_FALSE(glob_to_regex("int_[0-9", ®ex).ok()); - EXPECT_FALSE(glob_to_regex("a[\\]b", ®ex).ok()); -} - -TEST(VariantUtilTest, GlobMatchRe2) { - struct Case { - std::string glob; - std::string candidate; - bool expected; - }; - const std::vector cases = { - {"*", "", true}, - {"*", "abc", true}, - {"?", "a", true}, - {"?", "", false}, - {"a?b", "acb", true}, - {"a?b", "ab", false}, - {"a*b", "ab", true}, - {"a*b", "axxxb", true}, - {"a**b", "ab", true}, - {"a**b", "axxxb", true}, - {"?*", "", false}, - {"?*", "a", true}, - {"*?", "", false}, - {"*?", "a", true}, - {"a*b", "a/b", true}, - {"a.b", "a.b", true}, - {"a.b", "acb", false}, - {"a+b", "a+b", true}, - {"a{b}", "a{b}", true}, - {"a|b", "a|b", true}, - {"a|b", "ab", false}, - {"a(b)c", "a(b)c", true}, - {"a(b)c", "abc", false}, - {"a^b", "a^b", true}, - {"a^b", "ab", false}, - {"a$b", "a$b", true}, - {"a$b", "ab", false}, - {"a[b-d]e", "ace", true}, - {"a[b-d]e", "aee", false}, - {"a[\\]]b", "a]b", true}, - {"a[\\]]b", "a[b", false}, - {"a[\\!]b", "a!b", true}, - {"a[\\!]b", "a]b", false}, - {"[]", "a", false}, - {"[!]", "]", false}, - {"\\", "\\", true}, - {"\\*", "\\abc", false}, - {"a[!\\]]b", "aXb", true}, - {"a[!\\]]b", "a]b", false}, - {"a[]b", "aXb", false}, - {"a[[]b", "a[b", true}, - {R"(a\*b)", "a*b", true}, - {R"(a\?b)", "a?b", true}, - {R"(a\[b)", "a[b", true}, - {R"(abc\)", R"(abc\)", true}, - {"int_[0-9]", "int_1", true}, - {"int_[0-9]", "int_a", false}, - {"int_[!0-9]", "int_a", true}, - {"int_[!0-9]", "int_1", false}, - {"int_[^0-9]", "int_b", true}, - {"int_[^0-9]", "int_2", false}, - {R"(a[\-]b)", "a-b", true}, - {"", "", true}, - {"", "a", false}, - }; - - for (const auto& test_case : cases) { - bool matched = glob_match_re2(test_case.glob, test_case.candidate); - EXPECT_EQ(matched, test_case.expected) - << "pattern=" << test_case.glob << " candidate=" << test_case.candidate; - } - - EXPECT_FALSE(glob_match_re2("int_[0-9", "int_1")); - EXPECT_FALSE(glob_match_re2("a[\\]b", "a]b")); -} - -} // namespace doris::variant_util diff --git a/be/test/util/variant/variant_canonical_test.cpp b/be/test/util/variant/variant_canonical_test.cpp index 42662c7262cd1b..83f2189494f16c 100644 --- a/be/test/util/variant/variant_canonical_test.cpp +++ b/be/test/util/variant/variant_canonical_test.cpp @@ -384,6 +384,7 @@ void expect_scalar_equivalent(const VariantScalarRef& scalar_ref, const OwnedVal EXPECT_EQ(scalar_hashes.crc, encoded_hashes.crc); EXPECT_EQ(scalar_hashes.crc32c, encoded_hashes.crc32c); EXPECT_EQ(arena(scalar_ref), arena(encoded_value.ref())); + EXPECT_EQ(canonical_compare(scalar_ref, encoded_value.ref()), 0); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive typed/encoded parity matrix. @@ -548,6 +549,119 @@ TEST(VariantCanonicalTest, NumericEquivalenceMatrix) { VariantPrimitiveId::DOUBLE); } +TEST(VariantCanonicalTest, ComparisonPreservesEqualityAndNumericOrder) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_FALSE(canonical_equals(left.ref(), right.ref())); + }; + const auto expect_equal = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_EQ(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_EQ(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_TRUE(canonical_equals(left.ref(), right.ref())); + EXPECT_EQ(hashes(left.ref()).sip, hashes(right.ref()).sip); + }; + + expect_order(decimal_value(15, 1), integer_value(100, 1)); + expect_order(decimal_value(1, 1), double_value(0.1)); + expect_order(double_value(std::nextafter(1.0, 0.0)), integer_value(1, 1)); + expect_order(integer_value(1, 1), double_value(std::nextafter(1.0, 2.0))); + expect_order(decimal_value(15, 1), double_value(1.5)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-999999999999999999LL, 1)); + expect_order(double_value(std::numeric_limits::infinity()), + double_value(std::numeric_limits::quiet_NaN())); + + expect_equal(integer_value(42, 1), decimal_value(4200, 2)); + expect_equal(integer_value(42, 1), double_value(42.0)); + expect_equal(double_bits(0x7FF0000000000001ULL), double_bits(0xFFFABCDE12345678ULL)); +} + +TEST(VariantCanonicalTest, ContainerComparisonIsLexicographicAndTransitive) { + const OwnedValue array_a = array_value({integer_value(1, 1).value}); + const OwnedValue array_b = array_value({integer_value(1, 1).value, integer_value(2, 1).value}); + const OwnedValue array_c = array_value({integer_value(1, 1).value, integer_value(3, 1).value}); + EXPECT_LT(canonical_compare(array_a.ref(), array_b.ref()), 0); + EXPECT_LT(canonical_compare(array_b.ref(), array_c.ref()), 0); + EXPECT_LT(canonical_compare(array_a.ref(), array_c.ref()), 0); + + const OwnedValue object_a = object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0}); + const OwnedValue object_b = object_value({"a"}, true, {0}, {integer_value(2, 1).value}, {0}); + const OwnedValue object_c = object_value({"b"}, true, {0}, {integer_value(0, 1).value}, {0}); + EXPECT_LT(canonical_compare(object_a.ref(), object_b.ref()), 0); + EXPECT_LT(canonical_compare(object_b.ref(), object_c.ref()), 0); + EXPECT_LT(canonical_compare(object_a.ref(), object_c.ref()), 0); + EXPECT_GT(canonical_compare(object_c.ref(), object_a.ref()), 0); +} + +TEST(VariantCanonicalTest, ComparisonBoundariesAndCrossKindTransitivity) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + }; + __int128 max_38_digits = 1; + for (int digit = 0; digit < 38; ++digit) { + max_38_digits *= 10; + } + max_38_digits -= 1; + + // Scaling 38-digit magnitudes by 10^38 needs the 256-bit intermediate. + expect_order(decimal_value(max_38_digits, 38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(-max_38_digits, 0), decimal_value(-max_38_digits, 38)); + // Subnormal and large doubles against exact values at the decimal limits. + expect_order(double_value(std::numeric_limits::denorm_min()), decimal_value(1, 38)); + expect_order(double_value(1e38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), double_value(1.7e38)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), + double_value(std::numeric_limits::infinity())); + + // Numeric value first, canonical kind as the tie-breaker, must stay transitive. + const std::array chain {decimal_value(15, 1), double_value(1.5), + integer_value(2, 1), decimal_value(25, 1)}; + for (size_t left = 0; left < chain.size(); ++left) { + for (size_t right = left + 1; right < chain.size(); ++right) { + expect_order(chain[left], chain[right]); + } + } + // Numbers order before strings and containers. + expect_order(double_value(std::numeric_limits::quiet_NaN()), string_value("0")); + expect_order(string_value("0"), array_value({})); +} + +TEST(VariantCanonicalTest, ScalarAgainstEncodedRootMatchesEncodedComparison) { + // The mixed overload must order a scalar exactly as its encoded bytes would be ordered, for + // every kind of encoded root including the containers a scalar can never equal. + const std::string text = "b"; + const std::array scalars { + VariantScalarRef::null_value(), + VariantScalarRef::boolean(true), + VariantScalarRef::integer(2), + VariantScalarRef::decimal(15, 1), + VariantScalarRef::float64(1.5), + VariantScalarRef::float64(std::numeric_limits::quiet_NaN()), + VariantScalarRef::string(StringRef(text)), + VariantScalarRef::date(19000)}; + const std::array roots { + scalar(primitive(VariantPrimitiveId::NULL_VALUE)), + integer_value(2, 1), + decimal_value(15, 1), + double_value(1.5), + double_value(2.5), + string_value("a"), + string_value("b"), + array_value({integer_value(1, 1).value}), + object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0})}; + for (const VariantScalarRef& value : scalars) { + const OwnedValue encoded = scalar(physical(value)); + for (const OwnedValue& root : roots) { + EXPECT_EQ(canonical_compare(value, root.ref()), + canonical_compare(encoded.ref(), root.ref())); + } + } +} + TEST(VariantCanonicalTest, PrimitiveTypeClasses) { const OwnedValue null_a = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); const OwnedValue null_b = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); @@ -881,6 +995,11 @@ TEST(VariantCanonicalTest, RandomizedEqualsHashAndArenaProperties) { auto [left, right] = random_pair(random, index, expected_equal); const bool equal = canonical_equals(left.ref(), right.ref()); EXPECT_EQ(equal, expected_equal); + const int comparison = canonical_compare(left.ref(), right.ref()); + const int reverse_comparison = canonical_compare(right.ref(), left.ref()); + EXPECT_EQ(comparison == 0, equal); + EXPECT_EQ((comparison > 0) - (comparison < 0), + -((reverse_comparison > 0) - (reverse_comparison < 0))); const std::string left_arena = arena(left.ref()); const std::string right_arena = arena(right.ref()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 5229c6fc55d39a..76ce3dd54292aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -46,7 +46,6 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; -import org.apache.doris.common.GlobRegexUtil; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.DeepCopy; @@ -4116,11 +4115,12 @@ public Index getInvertedIndex(Column column, List subPath, String analyz String childName = child.getName(); if (child.getFieldPatternType() == PatternType.MATCH_NAME_GLOB) { try { - com.google.re2j.Pattern compiled = GlobRegexUtil.getOrCompilePattern(childName); - if (compiled.matcher(subPathString).matches()) { + java.nio.file.PathMatcher matcher = java.nio.file.FileSystems.getDefault() + .getPathMatcher("glob:" + childName); + if (matcher.matches(java.nio.file.Paths.get(subPathString))) { fieldPattern = childName; } - } catch (com.google.re2j.PatternSyntaxException | IllegalArgumentException e) { + } catch (Exception e) { continue; } } else if (child.getFieldPatternType() == PatternType.MATCH_NAME) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/stage/GlobPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/stage/GlobPatterns.java index 680c1126f72e6d..d217fc8faf55a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/stage/GlobPatterns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/stage/GlobPatterns.java @@ -45,9 +45,7 @@ * * *

Only what {@code StageUtil} calls is ported: hadoop's glob matching is not, since - * the compiled pattern is used here purely to validate. Note this is a different glob dialect - * from {@code org.apache.doris.common.GlobRegexUtil}, which treats {@code {} } as literal - * characters and has no notion of brace groups. + * the compiled pattern is used here purely to validate. */ final class GlobPatterns { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/GlobRegexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/GlobRegexUtil.java deleted file mode 100644 index ff0687d5cc10f1..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/GlobRegexUtil.java +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common; - -import com.google.re2j.Pattern; - -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Utility to convert a restricted glob pattern into a regex. - * - * Supported glob syntax: - * - '*' matches any sequence of characters - * - '?' matches any single character - * - '[...]' matches any character in the brackets - * - '[!...]' matches any character not in the brackets - * - '\\' escapes the next character - */ -public final class GlobRegexUtil { - // Small LRU to cap compiled pattern memory - private static final int REGEX_CACHE_CAPACITY = 256; - private static final Map REGEX_CACHE = new LinkedHashMap( - REGEX_CACHE_CAPACITY, 0.75f, true) { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > REGEX_CACHE_CAPACITY; - } - }; - - private GlobRegexUtil() { - } - - public static Pattern getOrCompilePattern(String globPattern) { - synchronized (REGEX_CACHE) { - Pattern cached = REGEX_CACHE.get(globPattern); - if (cached != null) { - return cached; - } - String regex = globToRegex(globPattern); - Pattern compiled = Pattern.compile(regex); - REGEX_CACHE.put(globPattern, compiled); - return compiled; - } - } - - public static String globToRegex(String pattern) { - StringBuilder regexBuilder = new StringBuilder(); - regexBuilder.append("^"); - boolean isEscaped = false; - int patternLength = pattern.length(); - for (int index = 0; index < patternLength; index++) { - char currentChar = pattern.charAt(index); - if (isEscaped) { - appendEscapedRegexChar(regexBuilder, currentChar); - isEscaped = false; - continue; - } - if (currentChar == '\\') { - isEscaped = true; - continue; - } - if (currentChar == '*') { - regexBuilder.append(".*"); - continue; - } - if (currentChar == '?') { - regexBuilder.append('.'); - continue; - } - if (currentChar == '[') { - int classIndex = index + 1; - boolean classClosed = false; - boolean isClassEscaped = false; - StringBuilder classBuffer = new StringBuilder(); - if (classIndex < patternLength - && (pattern.charAt(classIndex) == '!' || pattern.charAt(classIndex) == '^')) { - classBuffer.append('^'); - classIndex++; - } - for (; classIndex < patternLength; classIndex++) { - char classChar = pattern.charAt(classIndex); - if (isClassEscaped) { - classBuffer.append(classChar); - isClassEscaped = false; - continue; - } - if (classChar == '\\') { - isClassEscaped = true; - continue; - } - if (classChar == ']') { - classClosed = true; - break; - } - classBuffer.append(classChar); - } - if (!classClosed) { - throw new IllegalArgumentException("Unclosed character class in glob pattern: " + pattern); - } - regexBuilder.append('[').append(classBuffer).append(']'); - index = classIndex; - continue; - } - appendEscapedRegexChar(regexBuilder, currentChar); - } - if (isEscaped) { - appendEscapedRegexChar(regexBuilder, '\\'); - } - regexBuilder.append("$"); - return regexBuilder.toString(); - } - - private static void appendEscapedRegexChar(StringBuilder regexBuilder, char ch) { - switch (ch) { - case '.': - case '^': - case '$': - case '+': - case '*': - case '?': - case '(': - case ')': - case '|': - case '{': - case '}': - case '[': - case ']': - case '\\': - regexBuilder.append('\\').append(ch); - break; - default: - regexBuilder.append(ch); - break; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index 7f11df8aab026e..f9eb08046aeb6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -116,11 +116,15 @@ public static PushDownContext createPushDownContext(RuntimeFilterContext rfConte } /** - * A context is valid if probeExpr references exactly one input slot. + * A context is valid if probeExpr references exactly one input slot and both expressions + * have non-Variant types. Scalar runtime filters cannot express Variant canonical equality; + * explicit casts to supported scalar types can still produce runtime filters. * Invalid when probeExpr is a constant (e.g., from OneRowRelation) or multi-slot. */ public boolean isValid() { - return probeExpr.getInputSlots().size() == 1; + return !srcExpr.getDataType().isVariantType() + && !probeExpr.getDataType().isVariantType() + && probeExpr.getInputSlots().size() == 1; } public PushDownContext withNewProbeExpression(Expression newProbe) { @@ -147,6 +151,10 @@ public Boolean visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapS @Override public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) { + // Recheck types after the probe expression is rewritten through Projects/SetOps/joins. + if (ctx.srcExpr.getDataType().isVariantType() || ctx.probeExpr.getDataType().isVariantType()) { + return false; + } if (!scan.canPushDownRuntimeFilter()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index db263edb7750f0..8b2f19b8532c81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.Slot; @@ -164,14 +163,14 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } else if (plan instanceof LogicalSort) { LogicalSort sort = (LogicalSort) plan; for (OrderKey orderKey : sort.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType()) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } } else if (plan instanceof LogicalTopN) { LogicalTopN topN = (LogicalTopN) plan; for (OrderKey orderKey : topN.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType()) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } @@ -182,29 +181,24 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } WindowExpression windowExpression = (WindowExpression) ((Alias) a).child(); if (windowExpression.getOrderKeys().stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + orderKey -> orderKey.getDataType().isObjectType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } if (windowExpression.getPartitionKeys().stream().anyMatch(( - partitionKey -> partitionKey.getDataType().isObjectOrVariantType()))) { + partitionKey -> partitionKey.getDataType().isObjectType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } }); } else if (plan instanceof LogicalJoin) { LogicalJoin join = (LogicalJoin) plan; for (Expression conjunct : join.getHashJoinConjuncts()) { - if (containsVariantTypeOutsideCast(conjunct)) { - throw new AnalysisException("variant type could not in join equal conditions: " - + conjunct.toSql()); - } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { + if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( "varbinary type could not in join equal conditions: " + conjunct.toSql()); } } for (Expression conjunct : join.getMarkJoinConjuncts()) { - if (containsVariantTypeOutsideCast(conjunct)) { - throw new AnalysisException("variant type could not in join equal conditions: " + conjunct.toSql()); - } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { + if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( "varbinary type could not in join equal conditions: " + conjunct.toSql()); } @@ -212,23 +206,6 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } } - private boolean containsVariantTypeOutsideCast(Expression expr) { - return containsVariantTypeOutsideCast(expr, false); - } - - private boolean containsVariantTypeOutsideCast(Expression expr, boolean underCast) { - boolean nextUnderCast = underCast || (expr instanceof Cast && !expr.getDataType().isVariantType()); - if (!nextUnderCast && expr.getDataType().isVariantType()) { - return true; - } - for (Expression child : expr.children()) { - if (containsVariantTypeOutsideCast(child, nextUnderCast)) { - return true; - } - } - return false; - } - private void checkMatchIsUsedCorrectly(Plan plan) { for (Expression expression : plan.getExpressions()) { if (expression instanceof Match) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index a37a0b40d72103..ffcb5d6a7b7857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -103,8 +103,6 @@ import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TinyIntType; -import org.apache.doris.nereids.types.VariantField; -import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; @@ -162,7 +160,6 @@ protected Expression processCompoundNewChildren(CompoundPredicate cp, List x + v, arr)) -- v is not in the output if (bindSlotByThisScope(unboundSlot).isEmpty()) { - return enclosingAnalyzer.visitUnboundSlotOfLambdaBody(unboundSlot, context, this); + return enclosingAnalyzer.visitUnboundSlot(unboundSlot, context); } return super.visitUnboundSlot(unboundSlot, context); } @@ -511,22 +490,6 @@ protected boolean shouldPrioritizeRelationQualifier() { return lambdaAnalyzer.analyze(lambdaFunction, context); } - /** - * Bind a name of a lambda body that is not a lambda argument. The element_at chain around the name is - * visited by the lambda analyzer, so whether the variant cast of the name is suppressed is the state of - * the lambda analyzer, not the state of this analyzer, which may be in a chain around the lambda. - */ - private Expression visitUnboundSlotOfLambdaBody(UnboundSlot unboundSlot, ExpressionRewriteContext context, - ExpressionAnalyzer lambdaAnalyzer) { - int enclosingDepth = suppressVariantElementAtCastDepth; - suppressVariantElementAtCastDepth = lambdaAnalyzer.suppressVariantElementAtCastDepth; - try { - return visitUnboundSlot(unboundSlot, context); - } finally { - suppressVariantElementAtCastDepth = enclosingDepth; - } - } - /** Whether relation-qualified columns should be resolved across scopes before nested fields. */ protected boolean shouldPrioritizeRelationQualifier() { return true; @@ -829,91 +792,6 @@ protected Expression processCompoundNewChildren(CompoundPredicate cp, List 0) { - return elementAt; - } - Optional path = resolveVariantElementAtPath(elementAt); - if (!path.isPresent()) { - return expr; - } - VariantType variantType = (VariantType) path.get().root.getDataType(); - Optional matchingField = variantType.findMatchingField(path.get().path); - if (!matchingField.isPresent()) { - return expr; - } - DataType targetType = matchingField.get().getDataType(); - return new Cast(elementAt, targetType); - } - - private Optional resolveVariantElementAtPath(ElementAt elementAt) { - List segments = new ArrayList<>(); - Expression current = elementAt; - Expression root = null; - while (current instanceof ElementAt) { - ElementAt currentElementAt = (ElementAt) current; - Optional key = getVariantPathKey(currentElementAt.right()); - if (!key.isPresent()) { - return Optional.empty(); - } - segments.add(0, key.get()); - Expression left = currentElementAt.left(); - if (left instanceof Cast && !((Cast) left).isExplicitType()) { - left = ((Cast) left).child(); - } - current = left; - root = left; - } - if (root == null || !(root.getDataType() instanceof VariantType)) { - return Optional.empty(); - } - if (segments.isEmpty()) { - return Optional.empty(); - } - return Optional.of(new VariantElementAtPath(root, String.join(".", segments))); - } - - private Optional getVariantPathKey(Expression expr) { - if (expr instanceof StringLikeLiteral) { - return Optional.of(((StringLikeLiteral) expr).getStringValue()); - } - return Optional.empty(); - } - - private static final class VariantElementAtPath { - private final Expression root; - private final String path; - - private VariantElementAtPath(Expression root, String path) { - this.root = root; - this.path = path; - } - } - - private Expression maybeCastAliasExpression(Alias alias, ExpressionRewriteContext context) { - if (suppressVariantElementAtCastDepth > 0 || !isEnableVariantSchemaAutoCast(context)) { - return alias; - } - Expression child = alias.child(); - if (!(child instanceof ElementAt)) { - return alias; - } - Expression casted = wrapVariantElementAtWithCast(child); - if (casted == child) { - return alias; - } - return alias.withChildren(ImmutableList.of(casted)); - } - @Override public Expression visitNot(Not not, ExpressionRewriteContext context) { // maybe is `not subquery`, we should bind it first diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java index 762896a240844b..c61c4ff04dacb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java @@ -82,7 +82,7 @@ private static Plan eliminateWindow(LogicalWindow window) { WindowExpression windowExpression = (WindowExpression) alias.child(); List orderExpressions = windowExpression.getOrderKeys(); if (orderExpressions.stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + orderKey -> orderKey.getDataType().isObjectType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } List orderKeys = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java index 2d63961c96bb9f..18e99c3c7e7888 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java @@ -19,9 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.JsonType; import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.ComplexDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -44,9 +42,6 @@ static boolean isExplicitlyCastable(DataType signatureType, DataType realType) { /** isExplicitlyCastable */ static boolean isPrimitiveExplicitlyCastable(DataType signatureType, DataType realType) { - if (signatureType instanceof JsonType && realType instanceof VariantType) { - return false; - } if (signatureType instanceof AnyDataType || signatureType instanceof FollowToAnyDataType || signatureType.isAssignableFrom(realType)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java index 3d2c49c907b453..e748df1eb59aae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java @@ -172,10 +172,10 @@ public FunctionSignature computeSignature(FunctionSignature signature) { DataType expressionType = arguments.get(0).getDataType(); DataType sigType = signature.argumentsTypes.get(0); if (expressionType instanceof VariantType && sigType instanceof VariantType) { - // Preserve predefinedFields for schema template matching - VariantType originalType = (VariantType) expressionType; - signature = signature.withArgumentType(0, originalType); - signature = signature.withReturnType(originalType); + // only keep the variant max subcolumns count + VariantType variantType = new VariantType(((VariantType) expressionType).getVariantMaxSubcolumnsCount()); + signature = signature.withArgumentType(0, variantType); + signature = signature.withReturnType(variantType); } return super.computeSignature(signature); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java index 6345f29f036c81..e53465515d898b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java @@ -19,7 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.NullOrIdenticalSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -35,7 +35,7 @@ * to_json convert type to json */ public class JsonHash extends ScalarFunction - implements UnaryExpression, NullOrIdenticalSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(JsonType.INSTANCE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java index 87da92d609faf9..4b579bcb22abe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java @@ -100,7 +100,7 @@ public ToJson withChildren(List children) { @Override public List getSignatures() { DataType firstChildType = child(0).getDataType(); - if (firstChildType.isStructType() || firstChildType.isArrayType()) { + if (firstChildType.isStructType() || firstChildType.isArrayType() || firstChildType.isVariantType()) { return ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE).args(firstChildType)); } if (firstChildType.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java index 8253f620a8e281..0ad327118f0709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java @@ -18,13 +18,9 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.PatternType; -import org.apache.doris.common.GlobRegexUtil; import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; -import com.google.re2j.Pattern; -import com.google.re2j.PatternSyntaxException; - import java.util.Objects; /** @@ -72,35 +68,6 @@ public String getComment() { return comment; } - /** - * Check if the given field name matches this field's pattern. - * This method uses a restricted glob syntax converted to regex. - * - * Supported glob syntax: - * - '*' matches any sequence of characters - * - '?' matches any single character - * - '[...]' matches any character in the brackets - * - '[!...]' matches any character not in the brackets - * - '\\' escapes the next character - * - * @param fieldName the field name to check - * @return true if the field name matches the pattern - */ - public boolean matches(String fieldName) { - if (patternType == PatternType.MATCH_NAME) { - return pattern.equals(fieldName); - } - if (patternType != PatternType.MATCH_NAME_GLOB) { - return false; - } - try { - Pattern compiled = GlobRegexUtil.getOrCompilePattern(pattern); - return compiled.matcher(fieldName).matches(); - } catch (PatternSyntaxException | IllegalArgumentException e) { - return false; - } - } - public org.apache.doris.catalog.VariantField toCatalogDataType() { return new org.apache.doris.catalog.VariantField( pattern, dataType.toCatalogDataType(), comment, patternType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java index 346435bcb1827b..04a2e8fc372617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -261,22 +260,6 @@ public List getPredefinedFields() { return predefinedFields; } - /** - * Find the first matching VariantField for the given field name. - * The matching is done in definition order, so the first matching pattern wins. - * - * @param fieldName the field name to match - * @return Optional containing the matching VariantField, or empty if no match - */ - public Optional findMatchingField(String fieldName) { - for (VariantField field : predefinedFields) { - if (field.matches(fieldName)) { - return Optional.of(field); - } - } - return Optional.empty(); - } - public int getVariantMaxSubcolumnsCount() { return variantMaxSubcolumnsCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 2896f58fa1c2b7..8d0162cf76f050 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -229,11 +229,9 @@ public static Optional implicitCast(DataType input, DataType expected) } } return Optional.of(new StructType(newFields)); - } else if (input instanceof VariantType && expected instanceof JsonType) { - // JSON functions require users to make this representation change explicit. - return Optional.empty(); - } else if (input instanceof VariantType && (expected.isNumericType() || expected.isStringLikeType())) { - // variant could implicit cast to numric types and string like types + } else if (input instanceof VariantType + && (expected.isNumericType() || expected.isStringLikeType() || expected.isJsonType())) { + // variant could implicit cast to numric types, string like types and json return Optional.of(expected); } else { return implicitCastPrimitive(input, expected); @@ -1457,6 +1455,19 @@ private static Expression processComparisonPredicateInternal(ComparisonPredicate boolean leftIsVariant = left.getDataType().isVariantType(); boolean rightIsVariant = right.getDataType().isVariantType(); + // V2 equality is shared by scalar predicates and canonical hash join keys. Keep + // ordering and mixed Variant/scalar comparisons on their existing coercion paths. + if (leftIsVariant && rightIsVariant && comparisonPredicate instanceof EqualPredicate) { + return comparisonPredicate; + } + // A bare NULL has no type of its own, so it takes the Variant type of the other side and + // `v = NULL`, `v != NULL` and `v <=> NULL` become the Variant equality above. + if (comparisonPredicate instanceof EqualPredicate && leftIsVariant != rightIsVariant + && (leftIsVariant ? right : left).getDataType().isNullType()) { + DataType variantDataType = leftIsVariant ? left.getDataType() : right.getDataType(); + return comparisonPredicate.withChildren(castIfNotSameType(left, variantDataType), + castIfNotSameType(right, variantDataType)); + } boolean isDirectVariantSubpathScalarComparison = leftIsVariant != rightIsVariant && ((leftIsVariant && left instanceof ElementAt) || (rightIsVariant && right instanceof ElementAt)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6fb6656ce7670a..886c2b37fc3c7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -860,7 +860,6 @@ public String toString() { // Default is false, which means do not flatten nested when create table. @Deprecated public static final String ENABLE_VARIANT_FLATTEN_NESTED = "enable_variant_flatten_nested"; - public static final String ENABLE_VARIANT_SCHEMA_AUTO_CAST = "enable_variant_schema_auto_cast"; // CLOUD_VARIABLES_BEGIN public static final String CLOUD_CLUSTER = "cloud_cluster"; @@ -3544,15 +3543,6 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { ) public int defaultVariantMaxSubcolumnsCount = 2048; - @VarAttrDef.VarAttr( - name = ENABLE_VARIANT_SCHEMA_AUTO_CAST, - needForward = true, - affectQueryResultInExecution = true, - description = "Whether to enable schema-template-based auto cast for variant expressions. " - + "The default is true." - ) - public boolean enableVariantSchemaAutoCast = true; - @VarAttrDef.VarAttr( name = DEFAULT_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE, needForward = true, @@ -6469,10 +6459,6 @@ public boolean getEnableVariantFlattenNested() { return enableVariantFlattenNested; } - public boolean isEnableVariantSchemaAutoCast() { - return enableVariantSchemaAutoCast; - } - public void setProfileLevel(String profileLevel) { int profileLevelTmp = Integer.valueOf(profileLevel); if (profileLevelTmp < 1 || profileLevelTmp > 3) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 144ab256416dad..450ebecb1b885d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -158,7 +158,8 @@ public class VariableMgr { "enable_nereids_trace", "nereids_trace_event_mode", "shuffled_agg_node_ids", - "plan_nereids_dump"); + "plan_nereids_dump", + "enable_variant_schema_auto_cast"); private static boolean isRemovedSessionVar(String varName) { return varName != null && REMOVED_SESSION_VAR_NAMES.contains(varName.toLowerCase()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index d7a985c5b320a7..04e6f1d373f738 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -43,6 +43,14 @@ public void testTimestampNsVariantSubtype() { Assertions.assertTrue(Type.getVariantSubTypes().contains(Type.TIMESTAMP_NS)); } + @Test + public void testTimestampTzIsNotVariantSubtype() { + Assertions.assertFalse(Type.VARIANT.supportSubType(ScalarType.createTimeStampTzType(6))); + Assertions.assertFalse(Type.VARIANT.supportSubType( + new ArrayType(ScalarType.createTimeStampTzType(3), true))); + Assertions.assertTrue(Type.VARIANT.supportSubType(ScalarType.createDatetimeV2Type(6))); + } + // ===================== ArrayType ===================== @Test public void testArrayOfArrayExactMatch() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GlobRegexUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GlobRegexUtilTest.java deleted file mode 100644 index 48f86c4e70c21a..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/common/GlobRegexUtilTest.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common; - -import com.google.re2j.Pattern; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - - -public class GlobRegexUtilTest { - - private void assertGlobToRegex(String globPattern, String expectedRegex) { - String regex = GlobRegexUtil.globToRegex(globPattern); - Assertions.assertEquals(expectedRegex, regex, "pattern: " + globPattern); - } - - @Test - public void testGlobToRegexBasicTokens() { - assertGlobToRegex("*", "^.*$"); - assertGlobToRegex("?", "^.$"); - assertGlobToRegex("a?b", "^a.b$"); - assertGlobToRegex("a*b", "^a.*b$"); - } - - @Test - public void testGlobToRegexRepeatedWildcards() { - assertGlobToRegex("a**b", "^a.*.*b$"); - assertGlobToRegex("a??b", "^a..b$"); - assertGlobToRegex("?*", "^..*$"); - assertGlobToRegex("*?", "^.*.$"); - } - - - @Test - public void testGlobToRegexEscaping() { - assertGlobToRegex("a.b", "^a\\.b$"); - assertGlobToRegex("a+b", "^a\\+b$"); - assertGlobToRegex("a{b}", "^a\\{b\\}$"); - assertGlobToRegex("a\\*b", "^a\\*b$"); - assertGlobToRegex("a\\?b", "^a\\?b$"); - assertGlobToRegex("a\\[b", "^a\\[b$"); - assertGlobToRegex("abc\\", "^abc\\\\$"); - assertGlobToRegex("a|b", "^a\\|b$"); - assertGlobToRegex("a(b)c", "^a\\(b\\)c$"); - assertGlobToRegex("a^b", "^a\\^b$"); - assertGlobToRegex("a$b", "^a\\$b$"); - } - - @Test - public void testGlobToRegexCharacterClasses() { - assertGlobToRegex("int_[0-9]", "^int_[0-9]$"); - assertGlobToRegex("int_[!0-9]", "^int_[^0-9]$"); - assertGlobToRegex("int_[^0-9]", "^int_[^0-9]$"); - assertGlobToRegex("a[\\-]b", "^a[-]b$"); - assertGlobToRegex("a[b-d]e", "^a[b-d]e$"); - assertGlobToRegex("a[\\]]b", "^a[]]b$"); - assertGlobToRegex("a[\\!]b", "^a[!]b$"); - } - - @Test - public void testGlobToRegexEmptyPattern() { - assertGlobToRegex("", "^$"); - } - - - @Test - public void testGlobToRegexWeirdClasses() { - assertGlobToRegex("a[[]b", "^a[[]b$"); - assertGlobToRegex("a[]b", "^a[]b$"); - Assertions.assertThrows(IllegalArgumentException.class, - () -> GlobRegexUtil.globToRegex("a[\\]b")); - } - - @Test - public void testGlobToRegexUnclosedClass() { - Assertions.assertThrows(IllegalArgumentException.class, - () -> GlobRegexUtil.globToRegex("int_[0-9")); - } - - - @Test - public void testGlobToRegexMoreWeirdCases() { - assertGlobToRegex("[]", "^[]$"); - assertGlobToRegex("[!]", "^[^]$"); - assertGlobToRegex("[^]", "^[^]$"); - assertGlobToRegex("\\", "^\\\\$"); - assertGlobToRegex("\\*", "^\\*$"); - assertGlobToRegex("a\\*b", "^a\\*b$"); - assertGlobToRegex("a[!\\]]b", "^a[^]]b$"); - } - - @Test - public void testGetOrCompilePatternCache() { - Pattern first = GlobRegexUtil.getOrCompilePattern("num_*"); - Pattern second = GlobRegexUtil.getOrCompilePattern("num_*"); - Assertions.assertSame(first, second); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java index 3334615f4a2bf0..044817079d5cec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; @@ -267,6 +268,18 @@ private String evidence() { } } + @Test + public void testTargetVariantTypeParticipatesInTryParseEquality() { + StringLiteral json = new StringLiteral("{\"a\":1}"); + VariantType firstTarget = new VariantType(1); + VariantType secondTarget = new VariantType(2); + + TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); + TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); + Assertions.assertNotEquals(firstTryParse, secondTryParse); + Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); + } + @FunctionalInterface private interface LoadFixtureSupplier { LoadFixture get() throws Exception; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java index 82f3ecc6a44bc9..1a86e41a30e117 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java @@ -29,10 +29,12 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; +import org.apache.doris.nereids.processor.post.RuntimeFilterPushDownVisitor.PushDownContext; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -56,6 +58,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.RuntimeFilterId; @@ -86,6 +89,10 @@ public void runBeforeAll() throws Exception { connectContext.getSessionVariable().setEnableRuntimeFilterPrune(false); connectContext.getSessionVariable().expandRuntimeFilterByInnerJoin = false; connectContext.getSessionVariable().setDisableJoinReorder(true); + createTable("CREATE TABLE variant_rf_a (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); + createTable("CREATE TABLE variant_rf_b (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); } @Test @@ -879,4 +886,62 @@ public void testDecoupledRuntimeFilter() { connectContext.getSessionVariable().enableDecoupledRuntimeFilter = false; } + @Test + public void testVariantJoinKeyDoesNotGenerateRuntimeFilter() { + // Control: the INT key of the same tables produces a runtime filter in this setup. + Assertions.assertFalse(getRuntimeFilters( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.k = b.k").get().isEmpty()); + // Scalar runtime filters cannot express Variant canonical equality. + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v = b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v <=> b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v['id'] = b.v['id']", + "SELECT * FROM (SELECT CAST(k AS VARIANT) vv FROM variant_rf_a) x " + + "JOIN variant_rf_b b ON x.vv = b.v")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } + + @Test + public void testVariantPushDownContextIsInvalidBeforeTraversal() { + RuntimeFilterContext context = new RuntimeFilterContext( + connectContext.getSessionVariable(), RuntimeFilterId.createGenerator()); + AbstractPhysicalPlan builder = Mockito.mock(AbstractPhysicalPlan.class); + SlotReference scalar = new SlotReference("scalar", IntegerType.INSTANCE); + SlotReference variant = new SlotReference("variant", VariantType.INSTANCE); + Expression cast = new Cast(variant, IntegerType.INSTANCE); + for (TRuntimeFilterType type : ImmutableList.of( + TRuntimeFilterType.IN, TRuntimeFilterType.BLOOM, + TRuntimeFilterType.MIN_MAX, TRuntimeFilterType.IN_OR_BLOOM)) { + Assertions.assertFalse(PushDownContext.createPushDownContext( + context, builder, variant, scalar, type).isValid()); + Assertions.assertFalse(PushDownContext.createPushDownContext( + context, builder, scalar, variant, type).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContext( + context, builder, scalar, cast, type).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContext( + context, builder, cast, scalar, type).isValid()); + } + } + + @Test + public void testScalarCastOfVariantGeneratesRuntimeFilter() { + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b " + + "ON CAST(a.v AS INT) = CAST(b.v AS INT)", + "SELECT * FROM (SELECT CAST(v AS INT) vv FROM variant_rf_a) a " + + "JOIN variant_rf_b b ON a.vv = b.k")) { + Assertions.assertFalse(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } + + @Test + public void testVariantSetOperationDoesNotGenerateRuntimeFilter() { + for (String sql : ImmutableList.of( + "SELECT v FROM variant_rf_a INTERSECT SELECT v FROM variant_rf_b", + "SELECT v FROM variant_rf_a EXCEPT SELECT v FROM variant_rf_b")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java deleted file mode 100644 index fc93df8c11b87d..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java +++ /dev/null @@ -1,386 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.analysis; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.Scope; -import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.analyzer.UnboundSlot; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.Between; -import org.apache.doris.nereids.trees.expressions.Cast; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.InPredicate; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; -import org.apache.doris.nereids.trees.expressions.functions.agg.Count; -import org.apache.doris.nereids.trees.expressions.functions.agg.Max; -import org.apache.doris.nereids.trees.expressions.functions.agg.Min; -import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; -import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; -import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; -import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.types.ArrayType; -import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.VariantField; -import org.apache.doris.nereids.types.VariantType; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.List; - -public class ExpressionAnalyzerVariantAutoCastTest { - - @AfterEach - public void cleanup() { - ConnectContext.remove(); - } - - private CascadesContext createContext(boolean enableAutoCast) { - ConnectContext ctx = new ConnectContext(); - ctx.getSessionVariable().enableVariantSchemaAutoCast = enableAutoCast; - ctx.setThreadLocalInfo(); - return CascadesContext.initTempContext(); - } - - private Expression analyze(Expression expr, Scope scope, boolean enableAutoCast) { - CascadesContext cascadesContext = createContext(enableAutoCast); - ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope, cascadesContext, true, true); - return analyzer.analyze(expr); - } - - private SlotReference buildVariantSlot(VariantType variantType) { - return new SlotReference(new org.apache.doris.nereids.trees.expressions.ExprId(1), - "data", variantType, true, ImmutableList.of()); - } - - private VariantType buildVariantType() { - VariantField numField = new VariantField("num_*", BigIntType.INSTANCE, ""); - VariantField strField = new VariantField("str_*", StringType.INSTANCE, ""); - return new VariantType(ImmutableList.of(numField, strField)); - } - - private void assertCastElementAt(Expression expr) { - Assertions.assertTrue(expr instanceof Cast, "expect Cast wrapping ElementAt"); - Cast cast = (Cast) expr; - Assertions.assertTrue(cast.child() instanceof ElementAt, "cast child should be ElementAt"); - } - - @Test - public void testTargetVariantTypeParticipatesInTryParseEquality() { - StringLiteral json = new StringLiteral("{\"a\":1}"); - VariantType firstTarget = new VariantType(1); - VariantType secondTarget = new VariantType(2); - - TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); - TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); - Assertions.assertNotEquals(firstTryParse, secondTryParse); - Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); - } - - @Test - public void testSelectAutoCastElementAt() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Expression result = analyze(elementAt, scope, true); - assertCastElementAt(result); - } - - @Test - public void testSelectDotSyntaxAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - UnboundSlot unbound = new UnboundSlot("data", "num_a"); - Expression result = analyze(unbound, scope, true); - Assertions.assertTrue(result instanceof Alias); - Alias alias = (Alias) result; - assertCastElementAt(alias.child()); - } - - @Test - public void testWhereAutoCastComparison() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - GreaterThan predicate = new GreaterThan(elementAt, new BigIntLiteral(10)); - Expression result = analyze(predicate, scope, true); - - Assertions.assertTrue(result instanceof GreaterThan); - GreaterThan gt = (GreaterThan) result; - assertCastElementAt(gt.left()); - } - - @Test - public void testOrderByExpressionAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Expression result = analyze(elementAt, scope, true); - assertCastElementAt(result); - } - - @Test - public void testGroupByExpressionAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("str_name")); - Expression result = analyze(elementAt, scope, true); - assertCastElementAt(result); - } - - @Test - public void testAggregateFunctionAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Sum sum = new Sum(elementAt); - Expression result = analyze(sum, scope, true); - - Assertions.assertTrue(result instanceof Sum); - Sum analyzedSum = (Sum) result; - assertCastElementAt(analyzedSum.child()); - } - - @Test - public void testHavingAutoCastWithAggregate() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Sum sum = new Sum(elementAt); - GreaterThan having = new GreaterThan(sum, new BigIntLiteral(100)); - Expression result = analyze(having, scope, true); - - Assertions.assertTrue(result instanceof GreaterThan); - GreaterThan gt = (GreaterThan) result; - Assertions.assertTrue(gt.left() instanceof Sum); - Sum analyzedSum = (Sum) gt.left(); - assertCastElementAt(analyzedSum.child()); - } - - @Test - public void testNonLiteralKeyNoAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - SlotReference keySlot = new SlotReference(new ExprId(2), "col", StringType.INSTANCE, true, ImmutableList.of()); - Scope scope = new Scope(ImmutableList.of(slot, keySlot)); - - ElementAt elementAt = new ElementAt(slot, keySlot); - Expression result = analyze(elementAt, scope, true); - Assertions.assertTrue(result instanceof ElementAt); - Assertions.assertFalse(result instanceof Cast); - } - - @Test - public void testNoMatchingTemplateNoAutoCast() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("unknown")); - Expression result = analyze(elementAt, scope, true); - Assertions.assertTrue(result instanceof ElementAt); - Assertions.assertFalse(result instanceof Cast); - } - - @Test - public void testChainedPathOnlyOuterCast() { - VariantField nestedField = new VariantField("int_nested.level1_num_1", BigIntType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(nestedField)); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt inner = new ElementAt(slot, new StringLiteral("int_nested")); - ElementAt outer = new ElementAt(inner, new StringLiteral("level1_num_1")); - Expression result = analyze(outer, scope, true); - - Assertions.assertTrue(result instanceof Cast); - Cast cast = (Cast) result; - Assertions.assertTrue(cast.child() instanceof ElementAt); - ElementAt castChild = (ElementAt) cast.child(); - Assertions.assertTrue(castChild.left() instanceof ElementAt); - Assertions.assertFalse(castChild.left() instanceof Cast); - } - - @Test - public void testDotPathMergedAliasCast() { - VariantField nestedField = new VariantField("int_nested.level1_num_1", BigIntType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(nestedField)); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - UnboundSlot unbound = new UnboundSlot("data", "int_nested", "level1_num_1"); - Expression result = analyze(unbound, scope, true); - Assertions.assertTrue(result instanceof Alias); - Alias alias = (Alias) result; - assertCastElementAt(alias.child()); - } - - @Test - public void testExplicitCastStillAutoCastsInner() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Cast explicit = new Cast(elementAt, IntegerType.INSTANCE); - Expression result = analyze(explicit, scope, true); - - Assertions.assertTrue(result instanceof Cast); - Cast outer = (Cast) result; - Assertions.assertTrue(outer.child() instanceof Cast); - Cast inner = (Cast) outer.child(); - Assertions.assertTrue(inner.child() instanceof ElementAt); - } - - @Test - public void testWhereBetweenAndIn() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Between between = new Between(elementAt, new BigIntLiteral(10), new BigIntLiteral(20)); - Expression betweenResult = analyze(between, scope, true); - Assertions.assertTrue(betweenResult.containsType(Cast.class)); - Assertions.assertTrue(betweenResult.containsType(ElementAt.class)); - Assertions.assertTrue(betweenResult.collectFirst( - expr -> expr instanceof Cast && ((Cast) expr).child() instanceof ElementAt).isPresent()); - - ElementAt elementAtStr = new ElementAt(slot, new StringLiteral("str_name")); - InPredicate inPredicate = new InPredicate(elementAtStr, - ImmutableList.of(new StringLiteral("alice"), new StringLiteral("bob"))); - Expression inResult = analyze(inPredicate, scope, true); - Assertions.assertTrue(inResult.containsType(Cast.class)); - Assertions.assertTrue(inResult.containsType(ElementAt.class)); - Assertions.assertTrue(inResult.collectFirst( - expr -> expr instanceof Cast && ((Cast) expr).child() instanceof ElementAt).isPresent()); - } - - @Test - public void testAggregateMinMaxAvgCountDistinct() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Min min = new Min(elementAt); - Max max = new Max(elementAt); - Avg avg = new Avg(elementAt); - Count countDistinct = new Count(true, elementAt); - - Expression minResult = analyze(min, scope, true); - Assertions.assertTrue(minResult instanceof Min); - assertCastElementAt(((Min) minResult).child()); - - Expression maxResult = analyze(max, scope, true); - Assertions.assertTrue(maxResult instanceof Max); - assertCastElementAt(((Max) maxResult).child()); - - Expression avgResult = analyze(avg, scope, true); - Assertions.assertTrue(avgResult instanceof Avg); - assertCastElementAt(((Avg) avgResult).child()); - - Expression countResult = analyze(countDistinct, scope, true); - Assertions.assertTrue(countResult instanceof Count); - assertCastElementAt(((Count) countResult).child(0)); - } - - private SlotReference buildArraySlot() { - return new SlotReference(new ExprId(2), "arr", ArrayType.of(BigIntType.INSTANCE), true, ImmutableList.of()); - } - - private List collectCastElementAt(Expression expr) { - return expr.collectToList(node -> node instanceof Cast && ((Cast) node).child() instanceof ElementAt); - } - - @Test - public void testLambdaBodyElementAtChainSuppressesCastOfDottedPrefix() { - // array_map(x -> data.num_nested['l1']['l2'], arr) - // data is not a lambda argument, so the enclosing analyzer binds data.num_nested, but the element_at - // chain around it is in the lambda body: data.num_nested is a prefix of the path and is not cast to - // the type of 'num_*', same as outside a lambda - SlotReference data = buildVariantSlot(buildVariantType()); - Scope scope = new Scope(ImmutableList.of(data, buildArraySlot())); - - Expression chain = new ElementAt( - new ElementAt(new UnboundSlot("data", "num_nested"), new StringLiteral("l1")), - new StringLiteral("l2")); - Expression outsideLambda = analyze(chain, scope, true); - Assertions.assertTrue(collectCastElementAt(outsideLambda).isEmpty(), outsideLambda.toSql()); - - Lambda lambda = new Lambda(ImmutableList.of("x"), chain); - Expression result = analyze( - new UnboundFunction("array_map", ImmutableList.of(lambda, new UnboundSlot("arr"))), scope, true); - Assertions.assertTrue(collectCastElementAt(result).isEmpty(), result.toSql()); - } - - @Test - public void testLambdaBodyIsNotSuppressedByElementAtChainAroundLambda() { - // array_map(x -> array(data.num_a), arr)[1][1] - // the element_at chain is around the higher order function, data.num_a in the lambda body is not - // a part of it and is still cast - SlotReference data = buildVariantSlot(buildVariantType()); - Scope scope = new Scope(ImmutableList.of(data, buildArraySlot())); - - Lambda lambda = new Lambda(ImmutableList.of("x"), - new UnboundFunction("array", ImmutableList.of(new UnboundSlot("data", "num_a")))); - Expression arrayMap = new UnboundFunction("array_map", ImmutableList.of(lambda, new UnboundSlot("arr"))); - Expression result = analyze( - new ElementAt(new ElementAt(arrayMap, new BigIntLiteral(1)), new BigIntLiteral(1)), scope, true); - - Assertions.assertEquals(1, collectCastElementAt(result).size(), result.toSql()); - } - - @Test - public void testAutoCastDisabled() { - VariantType variantType = buildVariantType(); - SlotReference slot = buildVariantSlot(variantType); - Scope scope = new Scope(ImmutableList.of(slot)); - - ElementAt elementAt = new ElementAt(slot, new StringLiteral("num_a")); - Expression result = analyze(elementAt, scope, false); - - Assertions.assertTrue(result instanceof ElementAt); - Assertions.assertFalse(result instanceof Cast); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java index 64001a1dd8f076..5c06356dfa8113 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.thrift.TUniqueId; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; @@ -54,21 +55,35 @@ void testAllowedCanonicalHashContexts() { } @Test - void testRejectedVariantComparisons() { - assertVariantComparisonRejected("SELECT v = v FROM t1"); - assertVariantComparisonRejected("SELECT v != v FROM t1"); - assertVariantComparisonRejected("SELECT v <=> v FROM t1"); + void testRejectedVariantOrderingAndMixedComparisons() { assertVariantComparisonRejected("SELECT v = k FROM t1"); assertVariantComparisonRejected("SELECT v > v FROM t1"); - assertVariantComparisonRejected("SELECT * FROM t1 WHERE v = v"); - assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v"); - assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v <=> t2.v"); - assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v AND t1.k = t2.k"); - assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v OR t1.k = t2.k"); - assertVariantComparisonRejected("SELECT EXISTS(SELECT 1 FROM t2 WHERE t1.v = t2.v) FROM t1"); assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); } + @Test + void testVariantEquality() { + assertAllAccepted( + "SELECT v = v, v != v, v <=> v FROM t1", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", + "SELECT * FROM t1 LEFT JOIN t2 ON t1.v <=> t2.v", + "SELECT * FROM t1 FULL JOIN t2 ON t1.v = t2.v AND t1.k = t2.k", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v OR t1.k = t2.k", + "SELECT EXISTS(SELECT 1 FROM t2 WHERE t1.v = t2.v) FROM t1", + "SELECT * FROM t1 WHERE v IN (SELECT v FROM t2)", + "SELECT * FROM t1 WHERE v NOT IN (SELECT v FROM t2)", + "SELECT * FROM t1 JOIN t2 ON t1.v['id'] = t2.v['id']"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.k"); + } + + @Test + void testVariantOrdering() { + assertPlanAccepted("SELECT v FROM t1 ORDER BY v"); + assertPlanAccepted("SELECT v FROM t1 ORDER BY v LIMIT 10"); + assertPlanAccepted("SELECT row_number() OVER (ORDER BY v) FROM t1"); + } + @Test void testOtherMetricAndJsonBehaviorDoesNotChange() { assertRejected("SELECT MAX(v) FROM t1", "Doris hll, bitmap"); @@ -89,6 +104,11 @@ private void assertAllAccepted(String... sqlStatements) { .map(sql -> (Executable) () -> assertAccepted(sql))); } + private void assertPlanAccepted(String sql) { + connectContext.setQueryId(new TUniqueId(1, 1)); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).plan(sql), sql); + } + private void assertVariantComparisonRejected(String sql) { assertRejected(sql, "CAST to a concrete type first"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/VariantFieldMatchTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/VariantFieldMatchTest.java deleted file mode 100644 index 66289238e86414..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/VariantFieldMatchTest.java +++ /dev/null @@ -1,365 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.types; - -import org.apache.doris.thrift.TPatternType; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -/** - * Unit tests for VariantField pattern matching and VariantType field lookup. - */ -public class VariantFieldMatchTest { - - // ==================== VariantField.matches() tests ==================== - - @Test - public void testExactMatch() { - VariantField field = new VariantField("number_latency", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME.name()); - - Assertions.assertTrue(field.matches("number_latency")); - Assertions.assertFalse(field.matches("number_latency_ms")); - Assertions.assertFalse(field.matches("other_field")); - } - - @Test - public void testRegexMetaLiteralPatterns() { - VariantField pipe = new VariantField("a|b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(pipe.matches("a|b")); - Assertions.assertFalse(pipe.matches("ab")); - - VariantField paren = new VariantField("a(b)c", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(paren.matches("a(b)c")); - Assertions.assertFalse(paren.matches("abc")); - - VariantField caret = new VariantField("a^b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(caret.matches("a^b")); - Assertions.assertFalse(caret.matches("ab")); - - VariantField dollar = new VariantField("a$b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(dollar.matches("a$b")); - Assertions.assertFalse(dollar.matches("ab")); - - VariantField range = new VariantField("a[b-d]e", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(range.matches("ace")); - Assertions.assertFalse(range.matches("aee")); - - VariantField escapedRight = new VariantField("a[\\]]b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedRight.matches("a]b")); - Assertions.assertFalse(escapedRight.matches("a[b")); - - VariantField escapedBang = new VariantField("a[\\!]b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedBang.matches("a!b")); - Assertions.assertFalse(escapedBang.matches("a]b")); - } - - @Test - public void testExactMatchDoesNotTreatGlob() { - VariantField field = new VariantField("num_*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME.name()); - - Assertions.assertTrue(field.matches("num_*")); - Assertions.assertFalse(field.matches("num_a")); - } - - @Test - public void testGlobMatchSuffix() { - // Pattern: number_* should match number_latency, number_count, etc. - VariantField field = new VariantField("number_*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("number_latency")); - Assertions.assertTrue(field.matches("number_count")); - Assertions.assertTrue(field.matches("number_")); - Assertions.assertFalse(field.matches("string_message")); - Assertions.assertFalse(field.matches("numbering")); - } - - @Test - public void testGlobMatchPrefix() { - // Pattern: *_latency should match number_latency, string_latency, etc. - VariantField field = new VariantField("*_latency", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("number_latency")); - Assertions.assertTrue(field.matches("string_latency")); - Assertions.assertTrue(field.matches("_latency")); - Assertions.assertFalse(field.matches("latency_ms")); - } - - @Test - public void testGlobMatchMiddle() { - // Pattern: num_*_ms should match num_latency_ms, num_count_ms, etc. - VariantField field = new VariantField("num_*_ms", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("num_latency_ms")); - Assertions.assertTrue(field.matches("num_count_ms")); - Assertions.assertTrue(field.matches("num__ms")); - Assertions.assertFalse(field.matches("num_latency")); - Assertions.assertFalse(field.matches("number_latency_ms")); - } - - @Test - public void testGlobMatchAll() { - // Pattern: * should match everything - VariantField field = new VariantField("*", StringType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("anything")); - Assertions.assertTrue(field.matches("")); - Assertions.assertTrue(field.matches("a.b.c")); - } - - @Test - public void testRepeatedWildcardPatterns() { - VariantField doubleStar = new VariantField("a**b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(doubleStar.matches("ab")); - Assertions.assertTrue(doubleStar.matches("axxxb")); - - VariantField questionStar = new VariantField("?*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertFalse(questionStar.matches("")); - Assertions.assertTrue(questionStar.matches("a")); - - VariantField starQuestion = new VariantField("*?", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertFalse(starQuestion.matches("")); - Assertions.assertTrue(starQuestion.matches("a")); - } - - @Test - public void testGlobMatchWithDot() { - // Pattern: metrics.* should match metrics.score, metrics.count, etc. - VariantField field = new VariantField("metrics.*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("metrics.score")); - Assertions.assertTrue(field.matches("metrics.count")); - Assertions.assertFalse(field.matches("metricsXscore")); - Assertions.assertFalse(field.matches("metrics")); - } - - @Test - public void testGlobMatchDotLiteral() { - // '.' should be treated as literal in glob and escaped in regex - VariantField field = new VariantField("a.b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("a.b")); - Assertions.assertFalse(field.matches("acb")); - } - - @Test - public void testDefaultPatternTypeIsGlob() { - // Default constructor should use MATCH_NAME_GLOB - VariantField field = new VariantField("number_*", BigIntType.INSTANCE, ""); - - Assertions.assertTrue(field.matches("number_latency")); - } - - // ==================== VariantType.findMatchingField() tests ==================== - - @Test - public void testFindMatchingFieldSinglePattern() { - VariantField field = new VariantField("number_*", BigIntType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(field)); - - Optional result = variantType.findMatchingField("number_latency"); - Assertions.assertTrue(result.isPresent()); - Assertions.assertEquals(BigIntType.INSTANCE, result.get().getDataType()); - } - - @Test - public void testFindMatchingFieldMultiplePatterns() { - VariantField numberField = new VariantField("number_*", BigIntType.INSTANCE, ""); - VariantField stringField = new VariantField("string_*", StringType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(numberField, stringField)); - - // Test number pattern - Optional numberResult = variantType.findMatchingField("number_latency"); - Assertions.assertTrue(numberResult.isPresent()); - Assertions.assertEquals(BigIntType.INSTANCE, numberResult.get().getDataType()); - - // Test string pattern - Optional stringResult = variantType.findMatchingField("string_message"); - Assertions.assertTrue(stringResult.isPresent()); - Assertions.assertEquals(StringType.INSTANCE, stringResult.get().getDataType()); - } - - @Test - public void testFindMatchingFieldNoMatch() { - VariantField field = new VariantField("number_*", BigIntType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(field)); - - Optional result = variantType.findMatchingField("string_message"); - Assertions.assertFalse(result.isPresent()); - } - - @Test - public void testFindMatchingFieldFirstMatchWins() { - // When multiple patterns match, the first one should win - VariantField field1 = new VariantField("num*", BigIntType.INSTANCE, ""); - VariantField field2 = new VariantField("number_*", DoubleType.INSTANCE, ""); - VariantType variantType = new VariantType(ImmutableList.of(field1, field2)); - - Optional result = variantType.findMatchingField("number_latency"); - Assertions.assertTrue(result.isPresent()); - // First pattern "num*" should match, returning BigIntType - Assertions.assertEquals(BigIntType.INSTANCE, result.get().getDataType()); - } - - @Test - public void testFindMatchingFieldEmptyPredefinedFields() { - VariantType variantType = new VariantType(0); - - Optional result = variantType.findMatchingField("any_field"); - Assertions.assertFalse(result.isPresent()); - } - - // ==================== Escape sequence tests (aligning with fnmatch behavior) ==================== - - @Test - public void testGlobEscapeAsterisk() { - // Pattern: int_\* should match literal "int_*", not "int_" followed by anything - VariantField field = new VariantField("int_\\*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("int_*")); - Assertions.assertFalse(field.matches("int_nested")); - Assertions.assertFalse(field.matches("int_")); - } - - @Test - public void testGlobEscapeQuestionMark() { - // Pattern: int_\? should match literal "int_?", not "int_" followed by any single char - VariantField field = new VariantField("int_\\?", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("int_?")); - Assertions.assertFalse(field.matches("int_1")); - Assertions.assertFalse(field.matches("int_")); - } - - @Test - public void testGlobEscapeBracket() { - // Pattern: int_\[ should match literal "int_[" - VariantField field = new VariantField("int_\\[", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("int_[")); - Assertions.assertFalse(field.matches("int_a")); - } - - @Test - public void testGlobEscapeBackslash() { - // Pattern: int_\\ should match literal "int_\" - VariantField field = new VariantField("int_\\\\", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("int_\\")); - Assertions.assertFalse(field.matches("int_")); - } - - @Test - public void testGlobUnclosedBracket() { - // No closing bracket: invalid glob for PathMatcher, expect no match - VariantField field = new VariantField("int_[0-9", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertFalse(field.matches("int_[0-9")); - Assertions.assertFalse(field.matches("int_1")); - } - - @Test - public void testWeirdGlobPatterns() { - VariantField emptyClass = new VariantField("a[]b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertFalse(emptyClass.matches("aXb")); - - VariantField escapedBracket = new VariantField("a[[]b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedBracket.matches("a[b")); - } - - @Test - public void testMoreWeirdGlobPatterns() { - VariantField emptyClass = new VariantField("[]", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertFalse(emptyClass.matches("a")); - Assertions.assertFalse(emptyClass.matches("")); - - VariantField negatedEmpty = new VariantField("[!]", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertFalse(negatedEmpty.matches("]")); - - VariantField escapedBackslash = new VariantField("\\", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedBackslash.matches("\\")); - - VariantField escapedStar = new VariantField("\\*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedStar.matches("*")); - Assertions.assertFalse(escapedStar.matches("\\\\abc")); - - VariantField escapedCharInClass = new VariantField("a[!\\]]b", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(escapedCharInClass.matches("aXb")); - Assertions.assertFalse(escapedCharInClass.matches("a]b")); - } - - @Test - public void testGlobWithSlashSeparator() { - // With glob->regex, '*' should match '/' - VariantField field = new VariantField("int_*", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - - Assertions.assertTrue(field.matches("int_nested")); - Assertions.assertTrue(field.matches("int_nested.level1")); // '.' is matched by '*' - Assertions.assertTrue(field.matches("int_nested/level1")); // '/' is matched by '*' - } - - @Test - public void testGlobCharacterClass() { - // Character class tests - VariantField field1 = new VariantField("int_[0-9]", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(field1.matches("int_1")); - Assertions.assertFalse(field1.matches("int_a")); - - // Negated character class with ! - VariantField field2 = new VariantField("int_[!0-9]", BigIntType.INSTANCE, "", - TPatternType.MATCH_NAME_GLOB.name()); - Assertions.assertTrue(field2.matches("int_a")); - Assertions.assertFalse(field2.matches("int_1")); - - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index b0aa9fe1c75669..23a9939e7bbf01 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -17,12 +17,14 @@ package org.apache.doris.nereids.util; +import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.expression.check.CheckCast; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -45,8 +47,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Greatest; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.NullIf; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToJson; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; @@ -767,17 +771,28 @@ public void testImplicitCastFromPrimitiveToString() { } @Test - public void testVariantToJsonImplicitCastRequiresExplicitCast() { - Assertions.assertTrue(TypeCoercionUtils.implicitCast( - VariantType.INSTANCE, JsonType.INSTANCE).isEmpty()); + public void testVariantToJsonImplicitCast() { + Assertions.assertEquals(JsonType.INSTANCE, + TypeCoercionUtils.implicitCast(VariantType.INSTANCE, JsonType.INSTANCE).get()); } @Test - public void testVariantToJsonFunctionSignatureRequiresExplicitCast() { - Assertions.assertFalse(ExplicitlyCastableSignature.isExplicitlyCastable( + public void testVariantToJsonFunctionSignature() { + Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( JsonType.INSTANCE, VariantType.INSTANCE)); } + @Test + public void testToJsonAcceptsVariantArgument() { + SlotReference variant = new SlotReference("v", new VariantType(100)); + FunctionSignature signature = new ToJson(variant).getSignature(); + Assertions.assertEquals(JsonType.INSTANCE, signature.returnType); + Assertions.assertEquals(variant.getDataType(), signature.getArgType(0)); + // JSON builders keep converting a Variant argument through to_json. + Expression array = new JsonArray(variant).rewriteWhenAnalyze(); + Assertions.assertEquals(new ToJson(variant), array.child(0)); + } + @Test public void testVariantExistingImplicitCastsArePreserved() { Assertions.assertEquals(IntegerType.INSTANCE, @@ -1054,13 +1069,26 @@ public void testVariantComparisonRequiresExplicitCast() { ElementAt variantSubpath = new ElementAt(variant, new StringLiteral("c")); ElementAt anotherVariantSubpath = new ElementAt(anotherVariant, new StringLiteral("c")); - AnalysisException equality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(variant, anotherVariant))); - Assertions.assertTrue(equality.getMessage().contains("CAST to a concrete type first")); - - AnalysisException nullSafeEquality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new NullSafeEqual(variant, anotherVariant))); - Assertions.assertTrue(nullSafeEquality.getMessage().contains("CAST to a concrete type first")); + // Variant equality uses canonical comparison, so it is accepted without a cast. + EqualTo equality = new EqualTo(variant, anotherVariant); + Assertions.assertSame(equality, TypeCoercionUtils.processComparisonPredicate(equality)); + + NullSafeEqual nullSafeEquality = new NullSafeEqual(variant, anotherVariant); + Assertions.assertSame(nullSafeEquality, TypeCoercionUtils.processComparisonPredicate(nullSafeEquality)); + + // A bare NULL takes the Variant type on either side; an ordering comparison stays rejected. + for (ComparisonPredicate bareNull : ImmutableList.of( + new EqualTo(variant, NullLiteral.INSTANCE), new EqualTo(NullLiteral.INSTANCE, variant), + new NullSafeEqual(variant, NullLiteral.INSTANCE), + new NullSafeEqual(NullLiteral.INSTANCE, variant))) { + Expression coerced = TypeCoercionUtils.processComparisonPredicate(bareNull); + Assertions.assertEquals(bareNull.getClass(), coerced.getClass()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(0).getDataType()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(1).getDataType()); + } + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate( + new GreaterThan(variant, NullLiteral.INSTANCE))); AnalysisException mixedType = Assertions.assertThrows(AnalysisException.class, () -> TypeCoercionUtils.processComparisonPredicate(new GreaterThan(variant, integer))); @@ -1081,9 +1109,8 @@ public void testVariantComparisonRequiresExplicitCast() { Assertions.assertEquals(subpathComparison.child(0).getDataType(), subpathComparison.child(1).getDataType()); - Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate( - new EqualTo(variantSubpath, anotherVariantSubpath))); + EqualTo subpathEquality = new EqualTo(variantSubpath, anotherVariantSubpath); + Assertions.assertSame(subpathEquality, TypeCoercionUtils.processComparisonPredicate(subpathEquality)); Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( new GreaterThan(new Cast(variant, IntegerType.INSTANCE), integer))); diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java index ee721348919f49..8cfcd5211291b9 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java @@ -300,7 +300,8 @@ public abstract class Type { variantSubTypes.add(DATEV2); variantSubTypes.add(DATETIMEV2); variantSubTypes.add(TIMESTAMP_NS); - variantSubTypes.add(TIMESTAMP_TZ); + // TIMESTAMP_TZ is deliberately absent. This list is only checked when a column or + // function type is defined, so tables that already declare a TIMESTAMPTZ path keep working. variantSubTypes.add(IPV4); variantSubTypes.add(IPV6); variantSubTypes.add(UUID); diff --git a/regression-test/data/query_p0/join/test_join_on.out b/regression-test/data/query_p0/join/test_join_on.out index b14a14dcc75a94..0b96665bbd0451 100644 --- a/regression-test/data/query_p0/join/test_join_on.out +++ b/regression-test/data/query_p0/join/test_join_on.out @@ -3,3 +3,7 @@ 1 [1, 2] \N \N 2 [2, 3] \N \N +-- !variant_join -- +1 3 +2 4 + diff --git a/regression-test/data/variant_p0/predefine/test_schema_template_auto_cast.out b/regression-test/data/variant_p0/predefine/test_schema_template_auto_cast.out deleted file mode 100644 index 0c8715ca070e37..00000000000000 --- a/regression-test/data/variant_p0/predefine/test_schema_template_auto_cast.out +++ /dev/null @@ -1,342 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !where_simple -- -2 -3 - --- !where_and -- -2 -4 - --- !where_or -- -1 -3 -4 - --- !where_between -- -2 -4 - --- !where_in -- -1 -3 -4 - --- !order_by -- -3 50 -2 30 -4 15 -1 10 - --- !order_by_expr -- -3 51 -2 31 -4 16 -1 11 - --- !topn -- -3 50 -2 30 - --- !select_arithmetic -- -1 30 -2 70 -3 110 -4 40 - --- !case_when -- -1 low -2 high -3 high -4 low - --- !order_by_alias_expr -- -30 -40 -70 -110 - --- !explicit_cast_select -- -10 -30 -50 -15 - --- !explicit_cast_where -- -2 -3 - --- !explicit_cast_order_by -- -3 -2 -4 -1 - --- !group_by -- -alice 25 -bob 30 -charlie 50 - --- !group_by_multi_agg -- -alice 10 15 2 -bob 30 30 1 -charlie 50 50 1 - --- !having -- -alice 25 -bob 30 -charlie 50 - --- !having_min -- -bob 30 -charlie 50 - --- !having_non_agg -- -bob 30 -charlie 50 - --- !order_by_alias -- -10 -15 -30 -50 - --- !order_by_alias_subquery -- -1 10 -4 15 -2 30 -3 50 - --- !group_by_alias_subquery -- -10 1 -15 1 -30 1 -50 1 - --- !order_by_alias_nested -- -10 -15 -30 -50 - --- !group_by_alias_nested -- -10 1 -15 1 -30 1 -50 1 - --- !window_partition_order -- -1 1 -2 1 -3 1 -4 2 - --- !window_sum -- -1 25 -2 30 -3 50 -4 25 - --- !window_sum_order -- -1 10 -2 30 -3 50 -4 25 - --- !agg_min_max -- -10 50 - --- !agg_count_distinct -- -3 - --- !join_on -- -1 first -2 second - --- !join_on_alias_subquery -- -1 first -2 second - --- !match_name_exact_where -- -2 - --- !match_name_glob_where -- -1 - --- !match_name_exact_order -- -1 -2 - --- !match_name_glob_order -- -1 -2 - --- !leaf_int1_select -- -1 -2 -1 -3 - --- !leaf_int1_add -- -2 -3 -2 -4 - --- !leaf_int_nested_nonleaf -- -true -true -true -true - --- !leaf_int_nested_chain_select -- -1011111 -2022222 -3033333 -4044444 - --- !leaf_int_nested_dot_select -- -1011111 -2022222 -3033333 -4044444 - --- !leaf_int_nested_deref_select -- -1011111 -2022222 -3033333 -4044444 - --- !leaf_int_nested_chain_add -- -1011112 -2022223 -3033334 -4044445 - --- !leaf_int_nested_dot_add -- -1011112 -2022223 -3033334 -4044445 - --- !leaf_int_nested_deref_add -- -1011112 -2022223 -3033334 -4044445 - --- !leaf_where_ok -- -1 -2 -3 -4 - --- !leaf_where_nonleaf -- - --- !leaf_where_mixed_1 -- -2 -3 -4 - --- !leaf_where_mixed_2 -- -2 -3 -4 - --- !leaf_where_mixed_3 -- -2 -3 -4 - --- !leaf_order_by_ok -- -1 -3 -2 -4 - --- !leaf_order_by_nonleaf -- -1 -2 -3 -4 - --- !leaf_order_by_mixed_1 -- -1 -2 -3 -4 - --- !leaf_order_by_mixed_2 -- -1 -2 -3 -4 - --- !leaf_order_by_paren_root -- -1 -2 -3 -4 - --- !leaf_group_by_ok -- -1 2 -2 1 -3 1 - --- !leaf_group_by_nonleaf -- -true 4 - --- !leaf_group_by_mixed -- -1011111 1 -2022222 1 -3033333 1 -4044444 1 - --- !leaf_having_ok -- -1 2 -2 2 -3 3 - --- !leaf_having_mixed -- -3033333 3033333 -4044444 4044444 - --- !glob_wild_match -- -2 3 - --- !glob_literal_match -- -2 - --- !nonleaf_auto_cast_on -- -true - --- !nonleaf_auto_cast_off -- -{"level1_num_1":1011111,"level1_num_2":102} - --- !explicit_cast_chain_select_2 -- -10 -30 -50 -15 - --- !explicit_cast_chain_where_3 -- -2 -3 - --- !explicit_cast_chain_order_by_4 -- -3 -2 -4 -1 - --- !explicit_cast_chain_group_having_4 -- -15 1 -30 1 -50 1 - --- !explicit_cast_chain_match_2 -- -1 -4 - --- !explicit_cast_chain_match_4 -- -1 -4 diff --git a/regression-test/data/variant_p0/test_variant_bool_path.out b/regression-test/data/variant_p0/test_variant_bool_path.out new file mode 100644 index 00000000000000..affc49e1129fe7 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_bool_path.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_values -- +1 true +2 false +3 true +4 false + +-- !sql_group -- +2 1 +2 2 + +-- !sql_equality -- +1 +3 + +-- !sql_variant_group -- +2 1 +2 2 +1 5 + +-- !sql_rowsets_values_before_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_before_compaction -- +2 1 +2 2 + +-- !sql_rowsets_values_after_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_after_compaction -- +2 1 +2 2 + +-- !sql_buckets_values -- +1 true +2 false +3 true +4 false +5 true +6 false +7 true +8 false + +-- !sql_buckets_group -- +4 1 +4 2 + diff --git a/regression-test/data/variant_p0/test_variant_join.out b/regression-test/data/variant_p0/test_variant_join.out new file mode 100644 index 00000000000000..2ce98242b952e0 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join.out @@ -0,0 +1,160 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !shuffle -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !null_safe -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + +-- !composite_null_safe -- +1 1 + +-- !left -- +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !full -- +\N 4 +\N 6 +\N 9 +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !semi -- +1 +2 +3 +4 +6 +8 +9 + +-- !anti -- +10 +5 +7 + +-- !in -- +1 +2 +3 +4 +6 +8 +9 + +-- !not_in -- + +-- !mark -- +1 true +10 \N +2 true +3 true +4 true +5 \N +6 true +7 \N +8 true +9 true + +-- !exists -- +1 true +10 false +2 true +3 true +4 true +5 false +6 true +7 false +8 true +9 true + +-- !subpath -- +9 8 + +-- !residual -- +1 1 +2 1 +2 2 +3 2 +3 3 +4 3 +4 4 +5 5 +6 5 +6 6 +7 7 +8 7 +8 8 +9 8 +9 9 + +-- !scalar_equality -- +1 true false false +10 false true false +2 true false false +3 false true false +4 false true false +5 \N \N true +6 false true false +7 false true false +8 false true false +9 false true false + +-- !json_null_equality -- +true false + +-- !not_in_non_null -- +10 +7 + +-- !typed_key -- +1 1 + +-- !spill -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + diff --git a/regression-test/data/variant_p0/test_variant_join_type_properties.out b/regression-test/data/variant_p0/test_variant_join_type_properties.out new file mode 100644 index 00000000000000..7ce010d8b2605d --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join_type_properties.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 2 +4 4 + +-- !shuffle -- +1 1 +2 2 +4 4 + +-- !null_safe -- +1 1 +2 2 +3 3 +4 4 + +-- !subpath -- +4 4 + +-- !left_semi -- +1 +2 +4 + +-- !left_anti -- +3 +5 + +-- !mark -- +1 true +2 true +3 \N +4 true +5 \N + +-- !not_in -- +5 + +-- !doc_mode -- +1 1 +4 4 + +-- !computed_key -- +1 1 +1 5 +2 2 +4 4 + +-- !inferred_key -- +1 1 +2 2 + diff --git a/regression-test/data/variant_p0/test_variant_json_functions.out b/regression-test/data/variant_p0/test_variant_json_functions.out new file mode 100644 index 00000000000000..bbf43f503d8849 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_json_functions.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_set_replace_remove -- +1 {"a":1,"b":{"c":[1,2]},"s":"x","z":9} {"a":0,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"r"} {"b":{"c":[1,2]},"s":"x"} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz","z":9} {"a":0,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"r"} {"b":{"c":[3]},"n":{"m":true},"s":"yz"} +6 \N \N \N \N + +-- !extract -- +1 [1,2] x 1 1 1 \N false \N x 1 +2 [3] yz 2 2 2 true false \N yz 2 +3 \N \N \N \N \N \N \N "two" \N \N +4 \N \N \N \N \N \N \N \N \N \N +5 \N \N \N \N \N \N \N \N \N \N +6 \N \N \N \N \N \N \N \N \N \N + +-- !inspect -- +1 ["a", "b", "s"] 3 object true true "$.s" 2600325834747410660 {"a":1,"b.c":[1,2],"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} +2 ["a", "b", "n", "s"] 4 object true false \N 8328850723868929851 {"a":2,"b.c":[3],"n.m":true,"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} +3 \N 3 array false \N \N 9010800628419254104 [1,"two",{"three":3}] [1,"two",{"three":3}] [1,"two",{"three":3}] +4 \N 1 int false \N \N 8321545726482568344 42 42 42 +5 \N 1 string false \N \N -1017189637099138661 "abc" "abc" "abc" +6 \N \N \N \N \N \N \N \N \N \N + +-- !construct -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} [{"a":1,"b":{"c":[1,2]},"s":"x"},1] [{"a":1,"b":{"c":[1,2]},"s":"x"}] {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"b":{"c":[1,2]}} {"v":1} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"},1] [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}] {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"b":{"c":[3]}} {"v":2} +3 [1,"two",{"three":3}] [[1,"two",{"three":3}],1] [[1,"two",{"three":3}]] {"v":[1,"two",{"three":3}]} {"v":[1,"two",{"three":3}]} {"b":null} {"v":null} +4 42 [42,1] [42] {"v":42} {"v":42} {"b":null} {"v":null} +5 "abc" ["abc",1] ["abc"] {"v":"abc"} {"v":"abc"} {"b":null} {"v":null} +6 \N [null,1] [] {"v":null} {"v":null} {"b":null} {"v":null} + +-- !array_of_variant -- +3 [1,"two",{"three":3}] [1,"two",{"three":3}] [[1,"two",{"three":3}]] + +-- !same_as_explicit_cast -- +1 true true true true true true +2 true true true true true true +3 true true true true true true +4 true true true true true true +5 true true true true true true +6 true true true true true true + +-- !string_signature -- +1 1 "{\\"a\\":1,\\"b\\":{\\"c\\":[1,2]},\\"s\\":\\"x\\"}" {"a":1,"b":{"c":[1,2]},"s":"x"} +4 1 "42" 42 +5 0 "abc" abc + +-- !json_parse -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} +4 42 + +-- !typed_scalars -- +string "2026-01-02 03:04:05.000000" Decimal32 ["2026-01-02",1.50] + diff --git a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out index ff0c49b7bc445c..ceb60826c894fb 100644 --- a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out +++ b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out @@ -1,4 +1,106 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_order -- +2 +3 +1 + +-- !variant_topn -- +1 +3 + +-- !variant_window -- +1 3 +2 1 +3 2 + +-- !asc_nulls_last -- +10 +2 +3 +1 +4 +6 +7 +8 +9 +5 + +-- !desc_nulls_first -- +5 +9 +7 +8 +6 +4 +1 +2 +3 +10 + +-- !rank_peers -- +1 5 4 +2 3 3 +3 3 3 +4 6 5 +5 1 1 +6 7 6 +7 8 7 +8 8 7 +9 10 8 +10 2 2 + +-- !partition_topn -- +5 1 +10 2 +2 3 +3 4 + +-- !spill_sort -- +5 +10 +2 +3 +1 +4 +6 +7 +8 +9 + +-- !spill_sort_desc -- +9 +7 +8 +6 +4 +1 +2 +3 +10 +5 + +-- !bare_null_equals -- +2 \N \N \N \N +5 \N \N \N \N +6 \N \N \N \N + +-- !bare_null_safe_equals -- +5 + +-- !bare_null_safe_equals_reversed -- +5 + +-- !bare_null_safe_not_equals -- +1 +10 +2 +3 +4 +6 +7 +8 +9 + -- !explicit_cast_order -- 1 2 diff --git a/regression-test/data/variant_p0/test_variant_relational_corners.out b/regression-test/data/variant_p0/test_variant_relational_corners.out new file mode 100644 index 00000000000000..54b9646c89aded --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_relational_corners.out @@ -0,0 +1,342 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !constant_types -- +false false false false false + +-- !decimal_types -- +true false true true + +-- !extracted -- +1 1 false +2 1 false +3 1 false +5 0 false +6 -0 false +8 \N true +9 \N true +10 \N true +11 [1,2] false +12 [1,2] false +13 [2,1] false +14 {"a":1,"b":2} false +15 {"a":1,"b":2} false +16 [] false +17 \N true +18 9007199254740992 false +19 9007199254740993 false +20 9007199254740992 false +21 1.5 false +22 false + +-- !groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !distinct -- +11 + +-- !native_order -- +8 +9 +10 +17 +5 +6 +1 +2 +21 +18 +20 +19 +22 +3 +14 +15 +16 +11 +12 +13 + +-- !native_order_desc -- +13 +11 +12 +16 +14 +15 +3 +22 +19 +18 + +-- !roundtrip_types -- +1 false true false +2 false true false +3 false false false +8 \N \N \N +9 \N \N \N +17 \N \N \N + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !cast_precision -- +18 18 +18 19 +18 20 +19 18 +19 19 +19 20 +20 18 +20 19 +20 20 + +-- !cast_groups -- +1 3 + +-- !cast_order -- +5 0 +6 -0 +1 1 +2 1 +21 1.5 +18 9007199254740992 +19 9007199254740992 +20 9007199254740992 + +-- !numeric_promotion -- +1 9007199254740992 +2 9007199254740993 +3 9007199254740992 +4 1 +5 true +6 false +7 0 + +-- !numeric_groups -- +1 2 +2 1 +4 1 +5 1 +6 1 +7 1 + +-- !numeric_join -- +1 1 +1 3 +2 2 +3 1 +3 3 +4 4 +5 5 +6 6 +7 7 + +-- !spill_groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !spill_join -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + diff --git a/regression-test/data/variant_p0/test_variant_subfield_comparison.out b/regression-test/data/variant_p0/test_variant_subfield_comparison.out new file mode 100644 index 00000000000000..681672733562bc --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_subfield_comparison.out @@ -0,0 +1,610 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 \N \N \N \N true \N \N true true true +21 \N 1 \N \N false \N \N false true false +22 1 \N \N \N false \N \N false false true +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 \N \N \N \N true \N \N true true true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 \N \N \N \N true \N \N true true true +37 \N [] \N \N false \N \N false true false +38 \N \N \N \N true \N \N true true true +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +28 +29 +31 +33 +35 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +26 +27 +28 +29 +31 +33 +35 +36 +38 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +21 +24 +25 +26 +27 +28 +29 +31 +33 +35 +36 +37 +38 + +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 null null true false true true false true false false +21 null 1 false true false false true false false false +22 1 null false true false false true false false false +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 null \N \N \N false \N \N false false true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 {} {} true false true true false true false false +37 {} [] false true false false true false false false +38 {"x":null} {} false true false false true false false false +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +24 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !null_object_roundtrip -- +20 true false true false true true +21 true false false false false false +26 true false true true true false +36 true false true false true true +37 true false false false false false +38 true false true false true false + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false false true false false true false +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- + +-- !sql -- +1 +2 + +-- !sql -- +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !explicit_numeric_cast -- +1 true false false +2 false false true +3 true false false +4 false false true +5 true false false +6 false false true +7 true false false +8 true false false +9 true false false +10 true false false +11 true false false +12 true false false + +-- !typed_projection -- +1 1 1 true false true +2 1 1.5 false true false +3 0 -0 true false true +4 9007199254740993 9007199254740992 false true false +5 9007199254740992 9007199254740992 true false true +6 \N 1 \N \N false +7 1 \N \N \N false +8 \N \N \N \N true +9 \N \N \N \N true + +-- !sql -- +1 +3 +5 + +-- !sql -- +2 +4 + +-- !sql -- +1 +3 +5 +8 +9 + diff --git a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out b/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out deleted file mode 100644 index 0ad3d02aa156e8..00000000000000 --- a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_all -- -1 2003-12-21 10:29:00+08:00 2003-12-21 10:29:00+08:00 2003-12-21 10:29:00.000+08:00 2003-12-21 15:29:00.000+08:00 2003-12-21 15:29:00.000000+08:00 2003-12-21 07:29:00.000000+08:00 - --- !select_cast_ts0 -- -1 2003-12-21 10:29:00+08:00 2003-12-21 10:29:00.000+08:00 2003-12-21 15:29:00.000000+08:00 - diff --git a/regression-test/data/variant_p0/test_variant_window_partition.out b/regression-test/data/variant_p0/test_variant_window_partition.out new file mode 100644 index 00000000000000..c2c30815eb02b1 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_window_partition.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_by_subpath -- +1 1 1 +2 1 2 +3 2 1 +4 a 1 +5 true 1 +6 true 2 +7 1 3 +8 true 3 + +-- !partition_by_whole -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_rank -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_topn -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 + +-- !partition_mixed_keys -- +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 2 +8 2 + +-- !many_blocks_typed_keys -- +5000 12502500 4169167 + +-- !many_blocks_encoded_keys -- +5000 12502500 4169167 + diff --git a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out index 80fea8c68d269b..9e635b956753a9 100644 --- a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out +++ b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out @@ -1,4 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !equals -- +true + +-- !not_equals -- +false + +-- !null_safe_equals -- +true + +-- !equality_join -- +0 0 +1 1 + +-- !null_safe_equality_join -- +0 0 +1 1 + -- !group_by -- 0 2 1 2 @@ -36,3 +53,4 @@ false null 1 -- !explicit_cast_equality -- true true + diff --git a/regression-test/suites/query_p0/join/test_join_on.groovy b/regression-test/suites/query_p0/join/test_join_on.groovy index 577ff414a25cae..8d33f933d52948 100644 --- a/regression-test/suites/query_p0/join/test_join_on.groovy +++ b/regression-test/suites/query_p0/join/test_join_on.groovy @@ -17,40 +17,43 @@ suite("test_join_on", "query_p0") { withGlobalLock("enable_non_aggregate_table_state_types") { - setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) { + setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) { - sql "DROP TABLE IF EXISTS join_on" - sql """ - CREATE TABLE join_on ( - `k1` int(11) NULL, - d_array ARRAY , - hll_col HLL , - `k3` bitmap, - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 4 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - sql """insert into join_on values (1, [1, 2], hll_hash(1), bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402')); """ - sql """insert into join_on values (2, [2, 3], hll_hash(2), bitmap_from_string('2, 4, 6, 8, 10, 12, 14, 100, 19910812, 20150403')); """ - qt_sql """ select * from join_on order by k1; """ - test { - sql """ select * from join_on as j1 inner join join_on as j2 on j1.hll_col = j2.hll_col; """ - exception "errCode = 2" - } + sql "DROP TABLE IF EXISTS join_on" + sql """ + CREATE TABLE join_on ( + `k1` int(11) NULL, + d_array ARRAY , + hll_col HLL , + `k3` bitmap, + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """insert into join_on values (1, [1, 2], hll_hash(1), bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402')); """ + sql """insert into join_on values (2, [2, 3], hll_hash(2), bitmap_from_string('2, 4, 6, 8, 10, 12, 14, 100, 19910812, 20150403')); """ + qt_sql """ select * from join_on order by k1; """ + test { + sql """ select * from join_on as j1 inner join join_on as j2 on j1.hll_col = j2.hll_col; """ + exception "errCode = 2" + } - test { - sql """ select * from join_on as j1 inner join join_on as j2 on j1.k3 = j2.k3; """ - exception "errCode = 2" - } + test { + sql """ select * from join_on as j1 inner join join_on as j2 on j1.k3 = j2.k3; """ + exception "errCode = 2" + } - test { - sql """select * from (select cast('' as variant) as a) t1 join (select cast('' as variant) as a) t2 on t1.a = t2.a""" - exception "could not used in ComparisonPredicate (a = a)" - } - } + // Variant keys use canonical equality in join conditions. + order_qt_variant_join """ + select t1.id, t2.id + from (select 1 as id, cast('x' as variant) as a union all select 2, cast(1 as variant)) t1 + join (select 3 as id, parse_to_variant('"x"') as a union all select 4, parse_to_variant('1.0')) t2 + on t1.a = t2.a + """ + } } } diff --git a/regression-test/suites/query_p0/set/test_removed_session_variables.groovy b/regression-test/suites/query_p0/set/test_removed_session_variables.groovy index 4e39984f29769d..83f36d2dde0824 100644 --- a/regression-test/suites/query_p0/set/test_removed_session_variables.groovy +++ b/regression-test/suites/query_p0/set/test_removed_session_variables.groovy @@ -36,7 +36,8 @@ suite("test_removed_session_variables") { "enable_eliminate_sort_node", "drop_table_if_ctas_failed", "trace_nereids", - "enable_sync_mv_cost_based_rewrite"] + "enable_sync_mv_cost_based_rewrite", + "enable_variant_schema_auto_cast"] // Test 1: SET on removed variables silently no-ops (no exception), across SET syntaxes. for (String var : removedVars) { diff --git a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse.groovy b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse.groovy index 7fdb9886dc85b4..56136f2e2d42a5 100644 --- a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse.groovy +++ b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse.groovy @@ -39,7 +39,6 @@ suite("predefined_typed_to_sparse", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_segment_limit_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - sql """ set enable_variant_schema_auto_cast = false """ def count = new Random().nextInt(10) + 1 def load_json_data = {table_name, file_name -> diff --git a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy index 102dbb6a33a352..96d61ea804cf93 100644 --- a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy +++ b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy @@ -39,7 +39,6 @@ suite("test_predefine_typed_to_sparse_1shard", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_segment_limit_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - sql """ set enable_variant_schema_auto_cast = false """ def load_json_data = {table_name, file_name -> // load the json data diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy index f832ccc716d1f1..89da23cdf36951 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy @@ -107,6 +107,12 @@ suite("test_predefine_ddl", "p0") { sql """ alter table ${tableName} add column var3 variant<'ab' : string> NULL """ + // A Schema Template cannot declare TIMESTAMPTZ paths. + test { + sql """ alter table ${tableName} add column var4 variant<'ts' : timestamptz(3)> NULL """ + exception("VARIANT unsupported sub-type: timestamptz(3)") + } + // TODO(lihangyu) : uncomment // test { // sql """ alter table ${tableName} modify column var variant<'ab' : string> NULL """ @@ -202,6 +208,30 @@ suite("test_predefine_ddl", "p0") { exception("VARIANT unsupported sub-type: datetime") } + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : timestamptz(6) + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: timestamptz(6)") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + 'ab*' : array + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: array NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${tableName} values(1, ${variantV2Function}('{"num_a": 10, "num_b": 20, "str_name": "alice"}'))""" - sql """insert into ${tableName} values(2, ${variantV2Function}('{"num_a": 30, "num_b": 40, "str_name": "bob"}'))""" - sql """insert into ${tableName} values(3, ${variantV2Function}('{"num_a": 50, "num_b": 60, "str_name": "charlie"}'))""" - sql """insert into ${tableName} values(4, ${variantV2Function}('{"num_a": 15, "num_b": 25, "str_name": "alice"}'))""" - - // Simple WHERE - qt_where_simple """ SELECT id FROM ${tableName} - WHERE data['num_a'] > 20 ORDER BY id """ - - // AND condition - qt_where_and """ SELECT id FROM ${tableName} - WHERE data['num_a'] > 10 AND data['num_b'] < 50 - ORDER BY id """ - - // OR condition - qt_where_or """ SELECT id FROM ${tableName} - WHERE data['num_a'] > 40 OR data['str_name'] = 'alice' - ORDER BY id """ - - // BETWEEN condition - qt_where_between """ SELECT id FROM ${tableName} - WHERE data['num_a'] BETWEEN 15 AND 30 - ORDER BY id """ - - // IN condition - qt_where_in """ SELECT id FROM ${tableName} - WHERE data['str_name'] IN ('alice', 'charlie') - ORDER BY id """ - - // Test 2: ORDER BY with auto-cast - qt_order_by """ SELECT id, data['num_a'] FROM ${tableName} - ORDER BY data['num_a'] DESC """ - - // ORDER BY expression - qt_order_by_expr """ SELECT id, data['num_a'] + 1 AS n FROM ${tableName} - ORDER BY data['num_a'] + 1 DESC """ - - // Test 3: TopN (ORDER BY + LIMIT) - qt_topn """ SELECT id, data['num_a'] FROM ${tableName} - ORDER BY data['num_a'] DESC LIMIT 2 """ - - // Test 4: SELECT with auto-cast (arithmetic operations) - qt_select_arithmetic """ SELECT id, data['num_a'] + data['num_b'] as sum_val - FROM ${tableName} ORDER BY id """ - - // CASE WHEN with auto-cast - qt_case_when """ SELECT id, - CASE WHEN data['num_a'] > 20 THEN 'high' ELSE 'low' END AS level - FROM ${tableName} ORDER BY id """ - - // ORDER BY alias from expression - qt_order_by_alias_expr """ SELECT data['num_a'] + data['num_b'] AS sum_val FROM ${tableName} - ORDER BY sum_val """ - - // Explicit CAST should still trigger schema template auto cast - qt_explicit_cast_select """ SELECT CAST(data['num_a'] AS INT) FROM ${tableName} ORDER BY id """ - qt_explicit_cast_where """ SELECT id FROM ${tableName} - WHERE CAST(data['num_a'] AS INT) > 20 ORDER BY id """ - qt_explicit_cast_order_by """ SELECT id FROM ${tableName} - ORDER BY CAST(data['num_a'] AS INT) DESC """ - - // Test 5: GROUP BY with auto-cast - qt_group_by """ SELECT data['str_name'], SUM(data['num_a']) as total - FROM ${tableName} GROUP BY data['str_name'] ORDER BY data['str_name'] """ - - // GROUP BY with multiple aggregates - qt_group_by_multi_agg """ SELECT data['str_name'], - MIN(data['num_a']) AS min_a, MAX(data['num_a']) AS max_a, COUNT(*) AS cnt - FROM ${tableName} GROUP BY data['str_name'] ORDER BY data['str_name'] """ - - // Test 6: HAVING with auto-cast - qt_having """ SELECT data['str_name'], SUM(data['num_a']) as total - FROM ${tableName} GROUP BY data['str_name'] - HAVING SUM(data['num_a']) > 20 ORDER BY data['str_name'] """ - - // HAVING with MIN - qt_having_min """ SELECT data['str_name'], MIN(data['num_a']) AS min_a - FROM ${tableName} GROUP BY data['str_name'] - HAVING MIN(data['num_a']) >= 15 ORDER BY data['str_name'] """ - - // HAVING with non-aggregate expression on group key - qt_having_non_agg """ SELECT data['str_name'], SUM(data['num_a']) AS total - FROM ${tableName} GROUP BY data['str_name'] - HAVING data['str_name'] != 'alice' ORDER BY data['str_name'] """ - - // Test 7: ORDER BY with alias from project - qt_order_by_alias """ SELECT data['num_a'] AS num_a FROM ${tableName} - ORDER BY num_a """ - - // Test 8: ORDER BY with alias from subquery - qt_order_by_alias_subquery """ SELECT * FROM (SELECT id, data['num_a'] AS num_a FROM ${tableName}) t - ORDER BY num_a, id """ - - // Test 9: GROUP BY with alias from subquery - qt_group_by_alias_subquery """ SELECT num_a, COUNT(*) AS cnt - FROM (SELECT data['num_a'] AS num_a FROM ${tableName}) t - GROUP BY num_a ORDER BY num_a """ - - // ORDER BY with nested alias - qt_order_by_alias_nested """ SELECT * FROM ( - SELECT num_a FROM (SELECT data['num_a'] AS num_a FROM ${tableName}) s1 - ) s2 ORDER BY num_a """ - - // GROUP BY with nested alias - qt_group_by_alias_nested """ SELECT num_a, COUNT(*) AS cnt FROM ( - SELECT num_a FROM (SELECT data['num_a'] AS num_a FROM ${tableName}) s1 - ) s2 GROUP BY num_a ORDER BY num_a """ - - // Test 10: WINDOW partition/order by with auto-cast - qt_window_partition_order """ SELECT id, - row_number() OVER (PARTITION BY data['str_name'] ORDER BY data['num_a']) AS rn - FROM ${tableName} ORDER BY id """ - - // WINDOW aggregate - qt_window_sum """ SELECT id, - SUM(data['num_a']) OVER (PARTITION BY data['str_name']) AS s - FROM ${tableName} ORDER BY id """ - - // WINDOW partition + order by with both paths - qt_window_sum_order """ SELECT id, - SUM(data['num_a']) OVER (PARTITION BY data['str_name'] ORDER BY data['num_a']) AS s - FROM ${tableName} ORDER BY id """ - - // Aggregates without GROUP BY - qt_agg_min_max """ SELECT MIN(data['num_a']), MAX(data['num_a']) FROM ${tableName} """ - qt_agg_count_distinct """ SELECT COUNT(DISTINCT data['str_name']) FROM ${tableName} """ - - // Test 11: disable auto-cast should error in ORDER BY - sql """ set enable_variant_schema_auto_cast = false """ - test { - sql """ SELECT id FROM ${tableName} ORDER BY data['num_a'] """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column must use with specific function" - } - sql """ set enable_variant_schema_auto_cast = true """ - - sql "DROP TABLE IF EXISTS ${tableName}" - - // Test 12: JOIN ON with auto-cast - def leftTable = "test_variant_join_left" - def rightTable = "test_variant_join_right" - - sql "DROP TABLE IF EXISTS ${leftTable}" - sql "DROP TABLE IF EXISTS ${rightTable}" - - sql """CREATE TABLE ${leftTable} ( - `id` bigint NULL, - `data` variant<'key_*': BIGINT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """CREATE TABLE ${rightTable} ( - `id` bigint NULL, - `info` variant<'key_*': BIGINT, 'name_*': STRING> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${leftTable} values(1, ${variantV2Function}('{"key_id": 100}'))""" - sql """insert into ${leftTable} values(2, ${variantV2Function}('{"key_id": 200}'))""" - sql """insert into ${leftTable} values(3, ${variantV2Function}('{"key_id": 300}'))""" - - sql """insert into ${rightTable} values(1, ${variantV2Function}('{"key_id": 100, "name_val": "first"}'))""" - sql """insert into ${rightTable} values(2, ${variantV2Function}('{"key_id": 200, "name_val": "second"}'))""" - sql """insert into ${rightTable} values(3, ${variantV2Function}('{"key_id": 400, "name_val": "fourth"}'))""" - - qt_join_on """ SELECT l.id, r.info['name_val'] - FROM ${leftTable} l JOIN ${rightTable} r - ON l.data['key_id'] = r.info['key_id'] - ORDER BY l.id """ - - // Test 13: JOIN ON with alias from subquery - qt_join_on_alias_subquery """ SELECT l.id, r.name_val - FROM (SELECT id, data['key_id'] AS key_id FROM ${leftTable}) l - JOIN (SELECT id, info['key_id'] AS key_id, info['name_val'] AS name_val FROM ${rightTable}) r - ON l.key_id = r.key_id - ORDER BY l.id """ - - sql "DROP TABLE IF EXISTS ${leftTable}" - sql "DROP TABLE IF EXISTS ${rightTable}" - - // Test 14: MATCH_NAME and MATCH_NAME_GLOB - def exactTable = "test_variant_schema_auto_cast_exact" - sql "DROP TABLE IF EXISTS ${exactTable}" - sql """CREATE TABLE ${exactTable} ( - `id` bigint NULL, - `data` variant<'exact_key': BIGINT, 'glob_*': BIGINT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${exactTable} values(1, ${variantV2Function}('{"exact_key": 10, "glob_1": 20, "glob_2": 5}'))""" - sql """insert into ${exactTable} values(2, ${variantV2Function}('{"exact_key": 30, "glob_2": 40}'))""" - - qt_match_name_exact_where """ SELECT id FROM ${exactTable} - WHERE data['exact_key'] > 10 ORDER BY id """ - qt_match_name_glob_where """ SELECT id FROM ${exactTable} - WHERE data['glob_1'] >= 20 ORDER BY id """ - qt_match_name_exact_order """ SELECT id FROM ${exactTable} - ORDER BY data['exact_key'] """ - qt_match_name_glob_order """ SELECT id FROM ${exactTable} - ORDER BY data['glob_2'], id """ - - sql "DROP TABLE IF EXISTS ${exactTable}" - - // Test 15: leaf vs non-leaf path auto cast limitation - def leafTable = "test_variant_schema_auto_cast_leaf" - sql "DROP TABLE IF EXISTS ${leafTable}" - sql """CREATE TABLE ${leafTable} ( - `id` bigint NULL, - `data` variant<'int_*': BIGINT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${leafTable} values - (1, ${variantV2Function}('{"int_1": 1, "int_nested": {"level1_num_1": 1011111, "level1_num_2": 102}}')), - (2, ${variantV2Function}('{"int_1": 2, "int_nested": {"level1_num_1": 2022222, "level1_num_2": 202}}')), - (3, ${variantV2Function}('{"int_1": 1, "int_nested": {"level1_num_1": 3033333, "level1_num_2": 302}}')), - (4, ${variantV2Function}('{"int_1": 3, "int_nested": {"level1_num_1": 4044444, "level1_num_2": 402}}'))""" - - qt_leaf_int1_select """ SELECT data['int_1'] FROM ${leafTable} ORDER BY id """ - qt_leaf_int1_add """ SELECT data['int_1'] + 1 FROM ${leafTable} ORDER BY id """ - // still fails: FE can't distinguish leaf/non-leaf, may cast int_nested to int - qt_leaf_int_nested_nonleaf """ SELECT data['int_nested'] IS NULL FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_chain_select """ SELECT data['int_nested']['level1_num_1'] - FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_dot_select """ SELECT data['int_nested.level1_num_1'] FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_deref_select """ SELECT data.int_nested.level1_num_1 FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_chain_add """ SELECT data['int_nested']['level1_num_1'] + 1 - FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_dot_add """ SELECT data['int_nested.level1_num_1'] + 1 - FROM ${leafTable} ORDER BY id """ - qt_leaf_int_nested_deref_add """ SELECT data.int_nested.level1_num_1 + 1 - FROM ${leafTable} ORDER BY id """ - - // Non-select clauses: leaf vs non-leaf - qt_leaf_where_ok """ SELECT id FROM ${leafTable} - WHERE data['int_1'] > 0 ORDER BY id """ - qt_leaf_where_nonleaf """ SELECT id FROM ${leafTable} - WHERE data['int_nested'] > 0 ORDER BY id """ - qt_leaf_where_mixed_1 """ SELECT id FROM ${leafTable} - WHERE data['int_nested']['level1_num_1'] > 2000000 ORDER BY id """ - qt_leaf_where_mixed_2 """ SELECT id FROM ${leafTable} - WHERE data['int_nested.level1_num_1'] > 2000000 ORDER BY id """ - qt_leaf_where_mixed_3 """ SELECT id FROM ${leafTable} - WHERE data.int_nested.level1_num_1 > 2000000 ORDER BY id """ - qt_leaf_order_by_ok """ SELECT id FROM ${leafTable} - ORDER BY data['int_1'], id """ - qt_leaf_order_by_nonleaf """ SELECT id FROM ${leafTable} - ORDER BY data['int_nested'], id """ - qt_leaf_order_by_mixed_1 """ SELECT id FROM ${leafTable} - ORDER BY data['int_nested']['level1_num_1'] """ - qt_leaf_order_by_mixed_2 """ SELECT id FROM ${leafTable} - ORDER BY data['int_nested.level1_num_1'] """ - qt_leaf_order_by_paren_root """ SELECT id FROM ${leafTable} - ORDER BY data.int_nested.level1_num_1 """ - qt_leaf_group_by_ok """ SELECT data['int_1'], COUNT(*) AS cnt - FROM ${leafTable} GROUP BY data['int_1'] ORDER BY data['int_1'] """ - qt_leaf_group_by_nonleaf """ SELECT data['int_nested'] IS NULL, COUNT(*) AS cnt - FROM ${leafTable} GROUP BY data['int_nested'] ORDER BY data['int_nested'] """ - qt_leaf_group_by_mixed """ SELECT data['int_nested.level1_num_1'], COUNT(*) AS cnt - FROM ${leafTable} GROUP BY data['int_nested.level1_num_1'] - ORDER BY data['int_nested.level1_num_1'] """ - qt_leaf_having_ok """ SELECT data['int_1'], SUM(data['int_1']) AS total - FROM ${leafTable} GROUP BY data['int_1'] - HAVING SUM(data['int_1']) > 0 ORDER BY data['int_1'] """ - qt_leaf_having_mixed """ SELECT data['int_nested.level1_num_1'], SUM(data['int_nested.level1_num_1']) AS total - FROM ${leafTable} GROUP BY data['int_nested.level1_num_1'] - HAVING SUM(data['int_nested.level1_num_1']) > 3000000 - ORDER BY data['int_nested.level1_num_1'] """ - - sql "DROP TABLE IF EXISTS ${leafTable}" - - // Test 16: backslash escaping in schema template pattern - def globWildTable = "test_variant_schema_auto_cast_glob_wild" - def globLiteralTable = "test_variant_schema_auto_cast_glob_literal" - def globLiteralPattern = "a\\*b" // SQL sees a\*b, glob sees a\*b (literal *) - - sql "DROP TABLE IF EXISTS ${globWildTable}" - sql "DROP TABLE IF EXISTS ${globLiteralTable}" - - sql """CREATE TABLE ${globWildTable} ( - `id` bigint NULL, - `data` variant<'a*b': BIGINT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """CREATE TABLE ${globLiteralTable} ( - `id` bigint NULL, - `data` variant<'${globLiteralPattern}': BIGINT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${globWildTable} values(1, ${variantV2Function}('{\"a*b\": 1, \"axb\": 2}'))""" - sql """insert into ${globLiteralTable} values(1, ${variantV2Function}('{\"a*b\": 1, \"axb\": 2}'))""" - - // wildcard a*b matches both a*b and axb - qt_glob_wild_match """ SELECT data['a*b'] + 1 AS v1, data['axb'] + 1 AS v2 - FROM ${globWildTable} ORDER BY id """ - - // literal a\*b matches only a*b - qt_glob_literal_match """ SELECT data['a*b'] + 1 AS v1 FROM ${globLiteralTable} ORDER BY id """ - test { - sql """ SELECT data['axb'] + 1 FROM ${globLiteralTable} """ - exception "Cannot cast from variant" - } - - sql "DROP TABLE IF EXISTS ${globWildTable}" - sql "DROP TABLE IF EXISTS ${globLiteralTable}" - - - // Test 17: non-leaf path auto cast limitation - def nonleafTable = "test_variant_schema_auto_cast_nonleaf_limit" - sql "DROP TABLE IF EXISTS ${nonleafTable}" - sql """CREATE TABLE ${nonleafTable} ( - `id` int NULL, - `data` variant<'int_*': INT> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${nonleafTable} values( - 1, ${variantV2Function}('{"int_1": 1, "int_nested": {"level1_num_1": 1011111, "level1_num_2": 102}}'))""" - - // auto cast enabled: non-leaf path matches int_* and returns NULL - sql "set enable_variant_schema_auto_cast = true" - qt_nonleaf_auto_cast_on """ SELECT data['int_nested'] IS NULL FROM ${nonleafTable} ORDER BY id """ - - // auto cast disabled: return original object - sql "set enable_variant_schema_auto_cast = false" - qt_nonleaf_auto_cast_off """ SELECT data['int_nested'] FROM ${nonleafTable} ORDER BY id """ - - // restore default - sql "set enable_variant_schema_auto_cast = true" - sql "DROP TABLE IF EXISTS ${nonleafTable}" - - - // Test 18: multi-layer explicit cast chain (2~4), including MATCH clause - def castChainTable = "test_variant_schema_auto_cast_cast_chain" - sql "DROP TABLE IF EXISTS ${castChainTable}" - sql """CREATE TABLE ${castChainTable} ( - `id` bigint NULL, - `data` variant<'num_*': BIGINT, 'str_*': STRING> NOT NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - sql """insert into ${castChainTable} values(1, ${variantV2Function}('{\"num_a\": 10, \"num_b\": 20, \"str_name\": \"alice\"}'))""" - sql """insert into ${castChainTable} values(2, ${variantV2Function}('{\"num_a\": 30, \"num_b\": 40, \"str_name\": \"bob\"}'))""" - sql """insert into ${castChainTable} values(3, ${variantV2Function}('{\"num_a\": 50, \"num_b\": 60, \"str_name\": \"charlie\"}'))""" - sql """insert into ${castChainTable} values(4, ${variantV2Function}('{\"num_a\": 15, \"num_b\": 25, \"str_name\": \"alice\"}'))""" - - qt_explicit_cast_chain_select_2 """ SELECT CAST(CAST(data['num_a'] AS BIGINT) AS BIGINT) - FROM ${castChainTable} ORDER BY id """ - qt_explicit_cast_chain_where_3 """ SELECT id FROM ${castChainTable} - WHERE CAST(CAST(CAST(data['num_a'] AS BIGINT) AS BIGINT) AS BIGINT) > 20 ORDER BY id """ - qt_explicit_cast_chain_order_by_4 """ SELECT id FROM ${castChainTable} - ORDER BY CAST(CAST(CAST(CAST(data['num_b'] AS BIGINT) AS BIGINT) AS BIGINT) AS BIGINT) DESC, id """ - qt_explicit_cast_chain_group_having_4 """ SELECT - CAST(CAST(CAST(CAST(data['num_a'] AS BIGINT) AS BIGINT) AS BIGINT) AS BIGINT) AS v, COUNT(*) - FROM ${castChainTable} - GROUP BY CAST(CAST(CAST(CAST(data['num_a'] AS BIGINT) AS BIGINT) AS BIGINT) AS BIGINT) - HAVING CAST(CAST(CAST(CAST(data['num_a'] AS BIGINT) AS BIGINT) AS BIGINT) AS BIGINT) >= 15 - ORDER BY v """ - - sql """ set enable_match_without_inverted_index = true """ - qt_explicit_cast_chain_match_2 """ SELECT id FROM ${castChainTable} - WHERE CAST(CAST(data['str_name'] AS STRING) AS VARCHAR) MATCH 'alice' ORDER BY id """ - qt_explicit_cast_chain_match_4 """ SELECT id FROM ${castChainTable} - WHERE CAST(CAST(CAST(CAST(data['str_name'] AS STRING) AS VARCHAR) AS STRING) AS VARCHAR) MATCH 'alice' ORDER BY id """ - sql """ set enable_match_without_inverted_index = false """ - - sql "DROP TABLE IF EXISTS ${castChainTable}" - -} diff --git a/regression-test/suites/variant_p0/test_variant_bool_path.groovy b/regression-test/suites/variant_p0/test_variant_bool_path.groovy new file mode 100644 index 00000000000000..c527dd1987f0ef --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_bool_path.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A Variant path that holds only JSON booleans keeps them as booleans: they read back as +// true/false, and equality and grouping see two values. That must hold however the rows reach +// storage - in one load, one load per row, spread over several tablets - and must survive a +// full compaction. +suite("test_variant_bool_path", "p0") { + sql "drop table if exists test_variant_bool_path" + sql """ + create table test_variant_bool_path ( + id int, + var variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + sql """ + insert into test_variant_bool_path (id, var) values + (1, parse_to_variant('{"k": true}')), + (2, parse_to_variant('{"k": false}')), + (3, parse_to_variant('{"k": true}')), + (4, parse_to_variant('{"k": false}')), + (5, parse_to_variant('{"other": 1}')) + """ + qt_sql_values """ + select id, var['k'] from test_variant_bool_path where var['k'] is not null order by id + """ + qt_sql_group """ + select count(*) as cnt, min(id) as first_id + from test_variant_bool_path where var['k'] is not null + group by var['k'] order by first_id + """ + qt_sql_equality """ + select id from test_variant_bool_path where var['k'] = parse_to_variant('true') order by id + """ + qt_sql_variant_group """ + select count(*) as cnt, min(id) as first_id + from test_variant_bool_path group by var order by first_id + """ + + // One load per row, so every rowset carries the path alone, then a full compaction merges them. + sql "drop table if exists test_variant_bool_path_rowsets" + sql """ + create table test_variant_bool_path_rowsets ( + id int, + var variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1", "disable_auto_compaction" = "true") + """ + sql """insert into test_variant_bool_path_rowsets values (1, parse_to_variant('{"k": true}'))""" + sql """insert into test_variant_bool_path_rowsets values (2, parse_to_variant('{"k": false}'))""" + sql """insert into test_variant_bool_path_rowsets values (3, parse_to_variant('{"k": true}'))""" + sql """insert into test_variant_bool_path_rowsets values (4, parse_to_variant('{"k": false}'))""" + qt_sql_rowsets_values_before_compaction """ + select id, var['k'] from test_variant_bool_path_rowsets order by id + """ + qt_sql_rowsets_group_before_compaction """ + select count(*) as cnt, min(id) as first_id + from test_variant_bool_path_rowsets group by var['k'] order by first_id + """ + trigger_and_wait_compaction("test_variant_bool_path_rowsets", "full") + qt_sql_rowsets_values_after_compaction """ + select id, var['k'] from test_variant_bool_path_rowsets order by id + """ + qt_sql_rowsets_group_after_compaction """ + select count(*) as cnt, min(id) as first_id + from test_variant_bool_path_rowsets group by var['k'] order by first_id + """ + + // One load spread over several tablets. + sql "drop table if exists test_variant_bool_path_buckets" + sql """ + create table test_variant_bool_path_buckets ( + id int, + var variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 4 + properties ("replication_num" = "1") + """ + sql """ + insert into test_variant_bool_path_buckets values + (1, parse_to_variant('{"k": true}')), (2, parse_to_variant('{"k": false}')), + (3, parse_to_variant('{"k": true}')), (4, parse_to_variant('{"k": false}')), + (5, parse_to_variant('{"k": true}')), (6, parse_to_variant('{"k": false}')), + (7, parse_to_variant('{"k": true}')), (8, parse_to_variant('{"k": false}')) + """ + qt_sql_buckets_values """ + select id, var['k'] from test_variant_bool_path_buckets order by id + """ + qt_sql_buckets_group """ + select count(*) as cnt, min(id) as first_id + from test_variant_bool_path_buckets group by var['k'] order by first_id + """ +} diff --git a/regression-test/suites/variant_p0/test_variant_join.groovy b/regression-test/suites/variant_p0/test_variant_join.groovy new file mode 100644 index 00000000000000..bd22fadad9d52c --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_join.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_join", "p0,nonConcurrent") { + sql "DROP TABLE IF EXISTS variant_join_left" + sql "DROP TABLE IF EXISTS variant_join_right" + sql """CREATE TABLE variant_join_left (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + sql """CREATE TABLE variant_join_right (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_join_left VALUES + (1, parse_to_variant('1')), (2, parse_to_variant('1.0')), + (3, parse_to_variant('"1"')), (4, parse_to_variant('false')), + (5, NULL), (6, parse_to_variant('{"a":1,"b":[2,null]}')), + (7, parse_to_variant('[1,2]')), (8, parse_to_variant('true')), + (9, parse_to_variant('{"id":10}')), (10, parse_to_variant('{}'))""" + sql """INSERT INTO variant_join_right VALUES + (1, parse_to_variant('1.0')), (2, parse_to_variant('"1"')), + (3, parse_to_variant('false')), (4, NULL), + (5, parse_to_variant('{"b":[2.0,null],"a":1.0}')), + (6, parse_to_variant('[2,1]')), (7, parse_to_variant('true')), + (8, parse_to_variant('{"id":10.0}')), (9, parse_to_variant('[]'))""" + + order_qt_broadcast """SELECT l.id, r.id FROM variant_join_left l + JOIN [broadcast] variant_join_right r ON l.v = r.v""" + order_qt_shuffle """SELECT l.id, r.id FROM variant_join_left l + JOIN [shuffle] variant_join_right r ON l.v = r.v""" + order_qt_null_safe """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v <=> r.v""" + order_qt_composite_null_safe """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v <=> r.v AND l.id % 2 = r.id % 2""" + order_qt_left """SELECT l.id, r.id FROM variant_join_left l + LEFT JOIN variant_join_right r ON l.v = r.v""" + order_qt_full """SELECT l.id, r.id FROM variant_join_left l + FULL JOIN variant_join_right r ON l.v = r.v""" + order_qt_semi """SELECT l.id FROM variant_join_left l + LEFT SEMI JOIN variant_join_right r ON l.v = r.v""" + order_qt_anti """SELECT l.id FROM variant_join_left l + LEFT ANTI JOIN variant_join_right r ON l.v = r.v""" + order_qt_in """SELECT id FROM variant_join_left + WHERE v IN (SELECT v FROM variant_join_right)""" + order_qt_not_in """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right)""" + order_qt_mark """SELECT l.id, l.v IN (SELECT r.v FROM variant_join_right r) + FROM variant_join_left l""" + order_qt_exists """SELECT l.id, EXISTS(SELECT 1 FROM variant_join_right r WHERE l.v = r.v) + FROM variant_join_left l""" + order_qt_subpath """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v['id'] = r.v['id']""" + order_qt_residual """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v = r.v OR l.id = r.id""" + order_qt_scalar_equality """SELECT l.id, l.v = parse_to_variant('1.0'), + l.v != parse_to_variant('1.0'), l.v <=> CAST(NULL AS VARIANT) + FROM variant_join_left l""" + qt_json_null_equality """SELECT parse_to_variant('null') = parse_to_variant('null'), + parse_to_variant('null') = parse_to_variant('{}')""" + // Without a NULL on the build side, NOT IN depends on canonical key matching. + order_qt_not_in_non_null """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right WHERE v IS NOT NULL)""" + // A typed key from CAST(scalar AS VARIANT) against a stored Variant key. + order_qt_typed_key """SELECT l.id, r.id + FROM (SELECT id, CAST(CAST(id AS BIGINT) AS VARIANT) k FROM variant_join_left) l + JOIN variant_join_right r ON l.k = r.v""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + order_qt_spill """SELECT l.id, r.id FROM variant_join_left l + JOIN [shuffle] variant_join_right r ON l.v <=> r.v""" + test { + sql "SELECT * FROM variant_join_left l JOIN variant_join_right r ON l.v > r.v" + exception "CAST to a concrete type first" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy new file mode 100644 index 00000000000000..c20b722288dc7d --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_join_type_properties", "p0,nonConcurrent") { + // Variant join keys compare canonically. Keys whose VARIANT types differ only in + // properties (max subcolumns count, doc mode) must join like identical types. + sql "DROP TABLE IF EXISTS variant_join_props_a" + sql "DROP TABLE IF EXISTS variant_join_props_b" + sql "DROP TABLE IF EXISTS variant_join_props_c" + sql "DROP TABLE IF EXISTS variant_join_props_d" + sql """CREATE TABLE variant_join_props_a ( + id INT, v VARIANT, s STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_b ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_c ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_d ( + id INT, + v VARIANT, + w VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_join_props_a VALUES + (1, parse_to_variant('1'), '1.0'), (2, parse_to_variant('"x"'), '"x"'), + (3, NULL, NULL), (4, parse_to_variant('{"k":1,"j":[1,2]}'), '{"j":[1.0,2],"k":1.0}'), + (5, parse_to_variant('true'), '1')""" + sql """INSERT INTO variant_join_props_b VALUES + (1, parse_to_variant('1.0')), (2, parse_to_variant('"x"')), (3, NULL), + (4, parse_to_variant('{"j":[1,2.0],"k":1}')), (6, parse_to_variant('false'))""" + sql """INSERT INTO variant_join_props_c VALUES + (1, parse_to_variant('1')), (4, parse_to_variant('{"k":1.0,"j":[1,2]}'))""" + sql """INSERT INTO variant_join_props_d VALUES + (1, parse_to_variant('1'), parse_to_variant('1.0')), + (2, parse_to_variant('"x"'), parse_to_variant('"x"')), + (4, parse_to_variant('{"k":1}'), parse_to_variant('{"k":2}'))""" + + // Different max subcolumns counts. + order_qt_broadcast """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [broadcast] variant_join_props_b b ON a.v = b.v""" + order_qt_shuffle """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [shuffle] variant_join_props_b b ON a.v = b.v""" + order_qt_null_safe """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v <=> b.v""" + order_qt_subpath """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v['j'] = b.v['j']""" + order_qt_left_semi """SELECT a.id FROM variant_join_props_a a + LEFT SEMI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_left_anti """SELECT a.id FROM variant_join_props_a a + LEFT ANTI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_mark """SELECT a.id, a.v IN (SELECT b.v FROM variant_join_props_b b) + FROM variant_join_props_a a""" + order_qt_not_in """SELECT a.id FROM variant_join_props_a a + WHERE a.v NOT IN (SELECT b.v FROM variant_join_props_b b WHERE b.v IS NOT NULL)""" + // Different doc mode. + order_qt_doc_mode """SELECT a.id, c.id FROM variant_join_props_a a + JOIN variant_join_props_c c ON a.v = c.v""" + // A stored column against a computed Variant whose type has default properties. + order_qt_computed_key """SELECT l.id, r.id FROM variant_join_props_a l + JOIN variant_join_props_a r ON l.v = parse_to_variant(r.s)""" + // The optimizer infers a.v = b.v from d.v = a.v, d.w = b.v and d.v = d.w and uses it + // as a hash key between tables whose Variant properties differ. + sql "SET disable_join_reorder = true" + order_qt_inferred_key """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.id = b.id + JOIN variant_join_props_d d ON d.v = a.v AND d.w = b.v + WHERE d.v = d.w""" +} diff --git a/regression-test/suites/variant_p0/test_variant_json_functions.groovy b/regression-test/suites/variant_p0/test_variant_json_functions.groovy new file mode 100644 index 00000000000000..458733170301a2 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// JSON functions accept a Variant argument by casting it to JSON implicitly. +suite("test_variant_json_functions", "p0") { + sql "DROP TABLE IF EXISTS test_variant_json_functions" + sql """ + CREATE TABLE test_variant_json_functions (k INT, v VARIANT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO test_variant_json_functions + SELECT k, parse_to_variant(doc) FROM ( + SELECT 1 AS k, '{"a":1,"s":"x","b":{"c":[1,2]}}' AS doc + UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}' + UNION ALL SELECT 3, '[1,"two",{"three":3}]' + UNION ALL SELECT 4, '42' + UNION ALL SELECT 5, 'abc' + UNION ALL SELECT 6, NULL) t + """ + + // json_insert and json_set reject a scalar document, so they only run on object rows. + order_qt_insert_set_replace_remove """ + SELECT k, json_insert(v, '\$.z', 9), json_set(v, '\$.a', 0), json_replace(v, '\$.s', 'r'), + json_remove(v, '\$.a') + FROM test_variant_json_functions WHERE k IN (1, 2, 6) ORDER BY k + """ + order_qt_extract """ + SELECT k, json_extract(v, '\$.b.c'), json_extract_string(v, '\$.s'), json_extract_int(v, '\$.a'), + json_extract_bigint(v, '\$.a'), json_extract_double(v, '\$.a'), json_extract_bool(v, '\$.n.m'), + json_extract_isnull(v, '\$.a'), jsonb_extract(v, '\$[1]'), get_json_string(v, '\$.s'), + get_json_int(v, '\$.a') + FROM test_variant_json_functions ORDER BY k + """ + order_qt_inspect """ + SELECT k, json_keys(v), json_length(v), json_type(v, '\$'), json_exists_path(v, '\$.s'), + json_contains(v, '1', '\$.a'), json_search(v, 'one', 'x'), json_hash(v), + json_object_flatten(v), sort_json_object_keys(v), normalize_json_numbers_to_double(v) + FROM test_variant_json_functions ORDER BY k + """ + // Variant values nested into new JSON documents keep their JSON structure. + order_qt_construct """ + SELECT k, to_json(v), json_array(v, 1), json_array_ignore_null(v, NULL), json_object('v', v), + json_insert('{}', '\$.v', v), json_set('{}', '\$.b', v['b']), json_replace('{"v":0}', '\$.v', v['a']) + FROM test_variant_json_functions ORDER BY k + """ + + // Variant elements of an ARRAY convert the same way. + order_qt_array_of_variant """ + SELECT k, to_json(CAST(v AS ARRAY)), CAST(CAST(v AS ARRAY) AS JSON), + json_array(CAST(v AS ARRAY)) + FROM test_variant_json_functions WHERE k = 3 + """ + + // The implicit cast gives the same result as the explicit one. + order_qt_same_as_explicit_cast """ + SELECT k, + CAST(json_replace(v, '\$.s', 'r') AS STRING) <=> CAST(json_replace(CAST(v AS JSON), '\$.s', 'r') AS STRING), + CAST(json_extract(v, '\$.b') AS STRING) <=> CAST(json_extract(CAST(v AS JSON), '\$.b') AS STRING), + CAST(json_keys(v) AS STRING) <=> CAST(json_keys(CAST(v AS JSON)) AS STRING), + json_hash(v) <=> json_hash(CAST(v AS JSON)), + CAST(to_json(v) AS STRING) <=> CAST(CAST(v AS JSON) AS STRING), + CAST(json_array(v) AS STRING) <=> CAST(json_array(CAST(v AS JSON)) AS STRING) + FROM test_variant_json_functions ORDER BY k + """ + + // Functions with a string signature keep treating Variant as text, so an unparsable document + // that was stored as a Variant string is still reported as invalid JSON. + order_qt_string_signature """ + SELECT k, json_valid(v), json_quote(v), json_unquote(v) + FROM test_variant_json_functions WHERE k IN (1, 4, 5) ORDER BY k + """ + order_qt_json_parse """ + SELECT k, json_parse(v) FROM test_variant_json_functions WHERE k IN (1, 4) ORDER BY k + """ + + // Typed Variant scalars without a JSON counterpart become JSON strings. + order_qt_typed_scalars """ + SELECT json_type(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), '\$'), + json_extract(CAST(CAST('2026-01-02 03:04:05' AS DATETIME) AS VARIANT), '\$'), + json_type(CAST(CAST(1.50 AS DECIMAL(5, 2)) AS VARIANT), '\$'), + json_array(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), CAST(1.50 AS DECIMAL(5, 2))) + """ +} diff --git a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy index e147f9e49b378b..f44d4fca436564 100644 --- a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy +++ b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy @@ -20,45 +20,85 @@ suite("test_variant_ordering_comparison_error", "p0,nonConcurrent") { sql "SET enable_nereids_planner = true" sql "SET enable_fallback_to_original_planner = false" - test { - sql """ - SELECT v + qt_variant_order """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v + UNION ALL + SELECT 2 AS id, parse_to_variant('1') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v + ORDER BY v, id """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" - } - - test { - sql """ - SELECT v + qt_variant_topn """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v - LIMIT 1 + ORDER BY v DESC, id + LIMIT 2 """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" - } + qt_variant_window """ + SELECT id, row_number() OVER (ORDER BY v, id) + FROM ( + SELECT 1 AS id, parse_to_variant('2') AS v + UNION ALL + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v + ) t + ORDER BY id + """ + // Canonical peers (1, 1.0 and objects with reordered keys), SQL NULL, a JSON null (read back + // as {}) and a boolean. One bucket keeps the numbers ahead of the boolean in the segment. + sql "DROP TABLE IF EXISTS variant_ordering_peers" + sql """CREATE TABLE variant_ordering_peers (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_ordering_peers VALUES + (1, parse_to_variant('2')), (2, parse_to_variant('1')), (3, parse_to_variant('1.0')), + (4, parse_to_variant('"a"')), (5, NULL), (6, parse_to_variant('null')), + (7, parse_to_variant('{"k":1,"j":2}')), (8, parse_to_variant('{"j":2,"k":1.0}')), + (9, parse_to_variant('[1,2]')), (10, parse_to_variant('true'))""" + qt_asc_nulls_last "SELECT id FROM variant_ordering_peers ORDER BY v ASC NULLS LAST, id" + qt_desc_nulls_first "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS FIRST, id" + qt_rank_peers """SELECT id, rank() OVER (ORDER BY v), dense_rank() OVER (ORDER BY v) + FROM variant_ordering_peers ORDER BY id""" + qt_partition_topn """SELECT id, rn FROM ( + SELECT id, row_number() OVER (ORDER BY v, id) rn FROM variant_ordering_peers) t + WHERE rn <= 4 ORDER BY rn""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_sort "SELECT id FROM variant_ordering_peers ORDER BY v NULLS FIRST, id" + qt_spill_sort_desc "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS LAST, id" + sql "SET enable_force_spill = false" + sql "SET enable_spill = false" + // A bare NULL takes the Variant type on either side. `=` and `!=` are NULL for every row; + // `<=>` matches the SQL NULL row only. + order_qt_bare_null_equals """SELECT id, v = NULL, NULL = v, v != NULL, NULL != v + FROM variant_ordering_peers WHERE id IN (2, 5, 6)""" + order_qt_bare_null_safe_equals "SELECT id FROM variant_ordering_peers WHERE v <=> NULL" + order_qt_bare_null_safe_equals_reversed "SELECT id FROM variant_ordering_peers WHERE NULL <=> v" + order_qt_bare_null_safe_not_equals "SELECT id FROM variant_ordering_peers WHERE NOT (v <=> NULL)" test { - sql "SELECT CAST('2' AS VARIANT) > CAST('1' AS VARIANT)" + sql "SELECT id FROM variant_ordering_peers WHERE v > NULL" exception "CAST to a concrete type first" } test { - sql "SELECT CAST('2' AS VARIANT) > 1" + sql "SELECT CAST('2' AS VARIANT) > CAST('1' AS VARIANT)" exception "CAST to a concrete type first" } test { - sql "SELECT CAST('2' AS VARIANT) <=> CAST('1' AS VARIANT)" + sql "SELECT CAST('2' AS VARIANT) > 1" exception "CAST to a concrete type first" } diff --git a/regression-test/suites/variant_p0/test_variant_relational_corners.groovy b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy new file mode 100644 index 00000000000000..3809664ab80bb4 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_relational_corners", "p0,nonConcurrent") { + qt_constant_types """SELECT + parse_to_variant('true') = parse_to_variant('1'), + parse_to_variant('false') = parse_to_variant('0'), + parse_to_variant('null') = parse_to_variant('{}'), + parse_to_variant('{"k":{}}')['k'] IS NULL, + parse_to_variant('9007199254740993') = parse_to_variant('9007199254740992.0')""" + qt_decimal_types """SELECT + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) + = CAST(CAST(1.500 AS DECIMAL(10,3)) AS VARIANT), + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1.5'), + CAST(CAST(1.00 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1'), + CAST(CAST(-1 AS TINYINT) AS VARIANT) = CAST(CAST(-1 AS BIGINT) AS VARIANT)""" + sql "DROP TABLE IF EXISTS variant_relational_corners" + sql """CREATE TABLE variant_relational_corners (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + // Keep values inside objects so extraction exercises shredded subcolumns, + // mixed-type promotion and the missing-path validity map. Booleans are compared in + // variant_relational_numeric instead: in this bucket layout a segment can see a boolean + // before a number, and storage then widens the boolean to an integer. + sql """INSERT INTO variant_relational_corners VALUES + (1, parse_to_variant('{"k":1}')), + (2, parse_to_variant('{"k":1.0}')), + (3, parse_to_variant('{"k":"1"}')), + (5, parse_to_variant('{"k":0}')), + (6, parse_to_variant('{"k":-0.0}')), + (8, parse_to_variant('{"k":null}')), + (9, parse_to_variant('{}')), + (10, NULL), + (11, parse_to_variant('{"k":[1,2]}')), + (12, parse_to_variant('{"k":[1.0,2.0]}')), + (13, parse_to_variant('{"k":[2,1]}')), + (14, parse_to_variant('{"k":{"a":1,"b":2}}')), + (15, parse_to_variant('{"k":{"b":2.0,"a":1.0}}')), + (16, parse_to_variant('{"k":[]}')), + (17, parse_to_variant('{"k":{}}')), + (18, parse_to_variant('{"k":9007199254740992}')), + (19, parse_to_variant('{"k":9007199254740993}')), + (20, parse_to_variant('{"k":9007199254740992.0}')), + (21, parse_to_variant('{"k":1.5}')), + (22, parse_to_variant('{"k":""}'))""" + + qt_extracted """SELECT id, v['k'], v['k'] IS NULL + FROM variant_relational_corners ORDER BY id""" + qt_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_distinct "SELECT count(DISTINCT v['k']) FROM variant_relational_corners" + qt_native_order """SELECT id FROM variant_relational_corners + ORDER BY v['k'] NULLS FIRST, id""" + qt_native_order_desc """SELECT id FROM variant_relational_corners + ORDER BY v['k'] DESC NULLS LAST, id LIMIT 10""" + // Compare stored values to constants explicitly: generated output is an + // observation of round-trip behavior, not a definition of type coercion. + qt_roundtrip_types """SELECT id, v['k'] = parse_to_variant('true'), + v['k'] = parse_to_variant('1'), v['k'] = parse_to_variant('{}') + FROM variant_relational_corners WHERE id IN (1,2,3,8,9,17) + ORDER BY id""" + ["broadcast", "shuffle"].each { distribution -> + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] = r.v['k'] + ORDER BY l.id, r.id""" + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" + } + // CAST is intentionally a different oracle here: strings can merge with + // numbers and DOUBLE cannot distinguish adjacent integers above 2^53. + qt_cast_precision """SELECT l.id, r.id FROM variant_relational_corners l + JOIN variant_relational_corners r + ON CAST(l.v['k'] AS DOUBLE) = CAST(r.v['k'] AS DOUBLE) + WHERE l.id BETWEEN 18 AND 20 AND r.id BETWEEN 18 AND 20 + ORDER BY l.id, r.id""" + qt_cast_groups """SELECT min(id), count(*) FROM variant_relational_corners + WHERE id IN (1,2,3) GROUP BY CAST(v['k'] AS BIGINT) ORDER BY min(id)""" + qt_cast_order """SELECT id, CAST(v['k'] AS DOUBLE) + FROM variant_relational_corners WHERE id IN (1,2,5,6,18,19,20,21) + ORDER BY CAST(v['k'] AS DOUBLE), id""" + sql "DROP TABLE IF EXISTS variant_relational_numeric" + sql """CREATE TABLE variant_relational_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_numeric VALUES + (1, parse_to_variant('{"k":9007199254740992}')), + (2, parse_to_variant('{"k":9007199254740993}')), + (3, parse_to_variant('{"k":9007199254740992.0}')), + (4, parse_to_variant('{"k":1}')), + (5, parse_to_variant('{"k":true}')), + (6, parse_to_variant('{"k":false}')), + (7, parse_to_variant('{"k":0}'))""" + qt_numeric_promotion """SELECT id, v['k'] FROM variant_relational_numeric ORDER BY id""" + qt_numeric_groups """SELECT min(id), count(*) FROM variant_relational_numeric + GROUP BY v['k'] ORDER BY min(id)""" + qt_numeric_join """SELECT l.id, r.id FROM variant_relational_numeric l + JOIN variant_relational_numeric r ON l.v['k'] = r.v['k'] ORDER BY l.id,r.id""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_spill_join """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [shuffle] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" +} diff --git a/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy new file mode 100644 index 00000000000000..2e2790ff0680ef --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_subfield_comparison", "p0,nonConcurrent") { + // Keep the JSON text as well as its stored Variant representation: reading a + // subcolumn can promote types, while parsing the text preserves each input. + sql "DROP TABLE IF EXISTS variant_subfield_comparison" + sql """CREATE TABLE variant_subfield_comparison (id INT, doc STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_comparison (id, doc) VALUES + (1, '{"a":1,"b":1}'), + (2, '{"a":1,"b":2}'), + (3, '{"a":-1,"b":-1}'), + (4, '{"a":127,"b":128}'), + (5, '{"a":1,"b":1.0}'), + (6, '{"a":1,"b":1.5}'), + (7, '{"a":1.5,"b":1.5}'), + (8, '{"a":0,"b":-0.0}'), + (9, '{"a":9007199254740993,"b":9007199254740992.0}'), + (10, '{"a":9007199254740992,"b":9007199254740992.0}'), + (11, '{"a":"1","b":1}'), + (12, '{"a":"1.5","b":1.5}'), + (13, '{"a":"abc","b":"abc"}'), + (14, '{"a":"abc","b":"abd"}'), + (15, '{"a":"","b":""}'), + (16, '{"a":true,"b":true}'), + (17, '{"a":true,"b":false}'), + (18, '{"a":true,"b":1}'), + (19, '{"a":false,"b":0}'), + (20, '{"a":null,"b":null}'), + (21, '{"a":null,"b":1}'), + (22, '{"a":1,"b":null}'), + (23, '{"a":1}'), + (24, '{"b":1}'), + (25, '{}'), + (26, '{"a":null}'), + (27, NULL), + (28, '{"a":[1,2],"b":[1,2]}'), + (29, '{"a":[1,2],"b":[1.0,2.0]}'), + (30, '{"a":[1,2],"b":[2,1]}'), + (31, '{"a":[],"b":[]}'), + (32, '{"a":[1],"b":1}'), + (33, '{"a":[1,null,"x",[2]],"b":[1.0,null,"x",[2.0]]}'), + (34, '{"a":[null],"b":[]}'), + (35, '{"a":{"x":1,"y":2},"b":{"y":2.0,"x":1.0}}'), + (36, '{"a":{},"b":{}}'), + (37, '{"a":{},"b":[]}'), + (38, '{"a":{"x":null},"b":{}}'), + (39, '{"a":[1],"b":["1"]}'), + (40, '{"a":[{"x":1}],"b":[{"x":2}]}')""" + // Insert into a second table to retain both original text and parsed values. + sql "DROP TABLE IF EXISTS variant_subfield_values" + sql """CREATE TABLE variant_subfield_values (id INT, doc STRING, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_values + SELECT id, doc, parse_to_variant(doc) FROM variant_subfield_comparison""" + + // These are separate round-trip expectations, not an assertion that storage + // preserves JSON null and empty objects. Rows 20-27 and 36-38 expose the + // current difference between stored-path SQL NULL and parsed JSON values. + ["v", "parse_to_variant(doc)"].each { expression -> + qt_sql """SELECT id, x['a'], x['b'], + x['a'] = x['b'], x['a'] != x['b'], x['a'] <=> x['b'], + x['b'] = x['a'], x['b'] != x['a'], x['b'] <=> x['a'], + x['a'] IS NULL, x['b'] IS NULL + FROM (SELECT id, ${expression} x FROM variant_subfield_values) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b'] ORDER BY id""" + } + qt_sql """SELECT id FROM variant_subfield_values + WHERE NOT (${expression}['a'] = ${expression}['b']) ORDER BY id""" + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] = ${expression}['b'] + OR ${expression}['a'] IS NULL ORDER BY id""" + } + // TopN ordering is supported; relational ordering predicates still require CAST. + ["v", "parse_to_variant(doc)"].each { expression -> + [">", ">=", "<", "<="].each { op -> + test { + sql "SELECT ${expression}['a'] ${op} ${expression}['b'] FROM variant_subfield_values" + exception "CAST to a concrete type first" + } + test { + sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b']""" + exception "CAST to a concrete type first" + } + } + } + // Isolate the known storage round-trip differences from ordinary equality cases. + qt_null_object_roundtrip """SELECT id, + v['a'] IS NULL, parse_to_variant(doc)['a'] IS NULL, + v['b'] IS NULL, parse_to_variant(doc)['b'] IS NULL, + v['a'] <=> v['b'], + parse_to_variant(doc)['a'] <=> parse_to_variant(doc)['b'] + FROM variant_subfield_values WHERE id IN (20,21,26,36,37,38) ORDER BY id""" + + // JSON text cannot retain Decimal, date or IP physical types. Cast scalar + // columns to Variant directly to exercise these same-row comparison types. + sql "DROP TABLE IF EXISTS variant_comparison_scalar_sources" + sql """CREATE TABLE variant_comparison_scalar_sources + (id INT, a STRING, b STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + [ + ["DECIMAL(18,2)", "DECIMAL(38,3)", "1.50", "1.500", "1.501"], + ["DECIMAL(18,2)", "DOUBLE", "1.50", "1.5", "1.6"], + ["DECIMAL(18,2)", "BIGINT", "1.00", "1", "2"], + ["DATEV2", "DATEV2", "2024-02-29", "2024-02-29", "2024-03-01"], + ["DATETIMEV2(3)", "DATETIMEV2(6)", "2024-02-29 12:34:56.123", + "2024-02-29 12:34:56.123000", "2024-02-29 12:34:56.123001"], + ["IPV4", "IPV4", "10.0.0.2", "10.0.0.2", "10.0.0.10"], + ["IPV6", "IPV6", "2001:db8::1", "2001:0db8:0:0:0:0:0:1", "2001:db8::2"], + ["DOUBLE", "DOUBLE", "NaN", "NaN", "Infinity"], + ["DOUBLE", "DOUBLE", "Infinity", "Infinity", "-Infinity"] + ].each { spec -> + sql "TRUNCATE TABLE variant_comparison_scalar_sources" + sql """INSERT INTO variant_comparison_scalar_sources VALUES + (1, '${spec[2]}', '${spec[3]}'), (2, '${spec[2]}', '${spec[4]}'), + (3, NULL, '${spec[3]}'), (4, '${spec[2]}', NULL), (5, NULL, NULL)""" + def source = """SELECT id, CAST(CAST(a AS ${spec[0]}) AS VARIANT) a, + CAST(CAST(b AS ${spec[1]}) AS VARIANT) b + FROM variant_comparison_scalar_sources""" + qt_sql """SELECT id, a IS NULL, b IS NULL, + a = b, a != b, a <=> b, b = a, b != a, b <=> a + FROM (${source}) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql "SELECT id FROM (${source}) t WHERE a ${op} b ORDER BY id" + } + // Typed Variant values also order natively; cover both NULL placements. + qt_sql "SELECT id FROM (${source}) t ORDER BY b ASC NULLS LAST, id" + qt_sql "SELECT id FROM (${source}) t ORDER BY b DESC NULLS FIRST, id" + } + qt_explicit_numeric_cast """SELECT id, + CAST(v['a'] AS DOUBLE) = CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) > CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) < CAST(v['b'] AS DOUBLE) + FROM variant_subfield_values WHERE id BETWEEN 1 AND 12 ORDER BY id""" + + // Homogeneous paths exercise typed subcolumns with different numeric widths. + sql "DROP TABLE IF EXISTS variant_subfield_numeric" + sql """CREATE TABLE variant_subfield_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_numeric VALUES + (1, parse_to_variant('{"a":1,"b":1.0}')), + (2, parse_to_variant('{"a":1,"b":1.5}')), + (3, parse_to_variant('{"a":0,"b":-0.0}')), + (4, parse_to_variant('{"a":9007199254740993,"b":9007199254740992.0}')), + (5, parse_to_variant('{"a":9007199254740992,"b":9007199254740992.0}')), + (6, parse_to_variant('{"a":null,"b":1.0}')), + (7, parse_to_variant('{"a":1}')), + (8, parse_to_variant('{}')), + (9, NULL)""" + qt_typed_projection """SELECT id, v['a'], v['b'], + v['a'] = v['b'], v['a'] != v['b'], v['a'] <=> v['b'] + FROM variant_subfield_numeric ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_numeric + WHERE v['a'] ${op} v['b'] ORDER BY id""" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy b/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy deleted file mode 100644 index 390fc35c4d6363..00000000000000 --- a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Reproduces DORIS-25915: -// When VARIANT has typed paths that exceed variant_max_subcolumns_count and -// variant_enable_typed_paths_to_sparse=true, timestamptz values that fall to -// the sparse path are read back as only the timezone suffix (e.g. "+00:00") -// instead of the full timestamp. -suite("test_variant_timestamptz_sparse", "p0"){ - def variantV2Function = "parse_to_variant" - sql " set time_zone = '+08:00' " - sql " set default_variant_enable_doc_mode = false " - - sql "DROP TABLE IF EXISTS test_variant_timestamptz_sparse_repro" - - sql """ - CREATE TABLE test_variant_timestamptz_sparse_repro ( - pk int, - var VARIANT< - 'c1':bigint, - 'c2':bigint, - 'd1':date, - 'd2':date, - 'dt0':datetime(0), - 'dt0n':datetime(0), - 'dt3':datetime(3), - 'dt3n':datetime(3), - 'dt6':datetime(6), - 'dt6n':datetime(6), - 'ts0':timestamptz(0), - 'ts0n':timestamptz(0), - 'ts3':timestamptz(3), - 'ts3n':timestamptz(3), - 'ts6':timestamptz(6), - 'ts6n':timestamptz(6), - 'v1':string, - 'v2':string, - properties("variant_max_subcolumns_count" = "2","variant_enable_typed_paths_to_sparse" = "true") - > NULL, - INDEX c1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c1"), - INDEX c2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c2"), - INDEX d1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d1"), - INDEX d2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d2"), - INDEX dt0 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0"), - INDEX dt0n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0n"), - INDEX dt3 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3"), - INDEX dt3n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3n"), - INDEX dt6 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6"), - INDEX dt6n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6n"), - INDEX v1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v1"), - INDEX v2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v2") - ) ENGINE=OLAP - DUPLICATE KEY(pk) - DISTRIBUTED BY HASH(pk) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ - INSERT INTO test_variant_timestamptz_sparse_repro VALUES - (1, ${variantV2Function}('{"c1":1699516324,"c2":1690122421,"d1":"2024-02-25","d2":"2024-11-27","dt0":"2002-12-25 12:42:19","dt0n":"2015-02-05 02:36:13","dt3":"2015-02-22 02:09:01","dt3n":"2015-09-16 02:55:07","dt6":"2001-09-19 09:53:52","dt6n":"2003-12-21 02:29:00","ts0":"2003-12-21 02:29:00 +00:00","ts0n":"2003-12-21 02:29:00 +00:00","ts3":"2003-12-21 02:29:00 +00:00","ts3n":"2003-12-21 02:29:00 -05:00","ts6":"2003-12-21 02:29:00 -05:00","ts6n":"2003-12-21 02:29:00 +03:00","v1":"2024-12-10 10:16:19","v2":"2023-12-31 23:59:59"}')); - """ - - sql "SYNC" - - // Pre-fix: every column that fell to sparse returned just the timezone - // suffix ("+08:00"). Post-fix: each ts* path round-trips as a full timestamp. - qt_select_all """ - SELECT - pk, - CAST(var['ts0'] AS string) AS str_ts0, - CAST(var['ts0n'] AS string) AS str_ts0n, - CAST(var['ts3'] AS string) AS str_ts3, - CAST(var['ts3n'] AS string) AS str_ts3n, - CAST(var['ts6'] AS string) AS str_ts6, - CAST(var['ts6n'] AS string) AS str_ts6n - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ - - qt_select_cast_ts0 """ - SELECT - pk, - CAST(var['ts0'] AS timestamptz(0)) AS cast_ts0, - CAST(var['ts3'] AS timestamptz(3)) AS cast_ts3, - CAST(var['ts6'] AS timestamptz(6)) AS cast_ts6 - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ -} diff --git a/regression-test/suites/variant_p0/test_variant_window_partition.groovy b/regression-test/suites/variant_p0/test_variant_window_partition.groovy new file mode 100644 index 00000000000000..bfdd0c951f6165 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_window_partition.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// GROUP BY, ORDER BY, hash join and window ORDER BY accept Variant, and they all use the same +// canonical equality and comparison. Window PARTITION BY used to be the one remaining context that +// rejected it, so these queries pin that a Variant partition key groups rows by canonical value: +// values of different JSON kinds ("a", true, 1) never share a partition, and equal values do. +suite("test_variant_window_partition", "p0") { + sql "drop table if exists test_variant_window_partition" + + sql """ + create table test_variant_window_partition ( + id int, + v variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + + sql """ + insert into test_variant_window_partition values + (1, parse_to_variant('{"k": 1}')), + (2, parse_to_variant('{"k": 1}')), + (3, parse_to_variant('{"k": 2}')), + (4, parse_to_variant('{"k": "a"}')), + (5, parse_to_variant('{"k": true}')), + (6, parse_to_variant('{"k": true}')), + (7, parse_to_variant('{"k": 1}')), + (8, parse_to_variant('{"k": true}')) + """ + + // PARTITION BY a Variant sub-path. + order_qt_partition_by_subpath """ + select id, v['k'] as k, row_number() over (partition by v['k'] order by id) as rn + from test_variant_window_partition + """ + + // PARTITION BY the whole Variant value. + order_qt_partition_by_whole """ + select id, row_number() over (partition by v order by id) as rn + from test_variant_window_partition + """ + + // rank() with a Variant partition key. + order_qt_partition_rank """ + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + """ + + // CreatePartitionTopNFromWindow matches logicalFilter(logicalWindow()), so a PartitionTopN + // operator only appears when a filter sits above the window. That operator organises the + // Variant partition key as a serialized hash key on BE, which the queries above never reach. + order_qt_partition_topn """ + select id, rk from ( + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + ) t where rk <= 2 + """ + + // A Variant partition key combined with a scalar one. + order_qt_partition_mixed_keys """ + select id, row_number() over (partition by v['k'], id % 2 order by id) as rn + from test_variant_window_partition + """ + + // Once the analytic sink has buffered more than 256 input blocks it erases the rows it has + // consumed from its partition and order key columns. A small batch size reaches that with a + // few thousand rows; the keys are typed Variant (CAST of a number) and encoded Variant + // (parse_to_variant) so both layouts are erased. + sql "set batch_size = 8" + qt_many_blocks_typed_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select cast(number as variant) v, cast(number % 3 as variant) p + from numbers("number" = "5000")) t + ) r + """ + qt_many_blocks_encoded_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select parse_to_variant(cast(number as string)) v, + parse_to_variant(cast(number % 3 as string)) p + from numbers("number" = "5000")) t + ) r + """ +} diff --git a/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse.groovy b/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse.groovy index 7fdb9886dc85b4..56136f2e2d42a5 100644 --- a/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse.groovy +++ b/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse.groovy @@ -39,7 +39,6 @@ suite("predefined_typed_to_sparse", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_segment_limit_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - sql """ set enable_variant_schema_auto_cast = false """ def count = new Random().nextInt(10) + 1 def load_json_data = {table_name, file_name -> diff --git a/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse_1shard.groovy b/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse_1shard.groovy index 102dbb6a33a352..96d61ea804cf93 100644 --- a/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse_1shard.groovy +++ b/regression-test/suites/variant_p0/v2/predefine/predefined_typed_to_sparse_1shard.groovy @@ -39,7 +39,6 @@ suite("test_predefine_typed_to_sparse_1shard", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_segment_limit_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - sql """ set enable_variant_schema_auto_cast = false """ def load_json_data = {table_name, file_name -> // load the json data diff --git a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy index 2a7e1e19500ade..47a4c7bf468b6c 100644 --- a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy +++ b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy @@ -75,23 +75,14 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { contains "EXCEPT" } - test { - sql "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + // V2 canonical equality is supported in scalar and join contexts. + qt_equals "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_not_equals "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_null_safe_equals "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - test { - sql """ + order_qt_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -99,11 +90,8 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v = b.v """ - exception "CAST to a concrete type first" - } - test { - sql """ + order_qt_null_safe_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -111,8 +99,6 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v <=> b.v """ - exception "CAST to a concrete type first" - } order_qt_group_by """ SELECT CAST(v AS STRING), COUNT(*) diff --git a/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md new file mode 100644 index 00000000000000..100499e966d053 --- /dev/null +++ b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md @@ -0,0 +1,97 @@ +# VARIANT relational benchmark + +`relational_performance.groovy` extends the existing `variant_p2` GitHub Events +workload with GROUP BY, ORDER BY, and broadcast/shuffle equality JOIN. The load +phase runs `load.groovy` and verifies its expected 44,273,863 rows. Pass +`--stream-load` to read the same public hourly files over HTTPS when regression +S3 credentials are unavailable. HTTP reads use a socket timeout and resumable +download. If an interrupted run has committed all but a known set of files, +pass those names with `--resume-files`; this appends only those files before the +same exact row-count check and prepare phase. +It then creates one small dimension table per selected real JSON path: + +| Key | Source path | CAST comparison | +| --- | --- | --- | +| actor_login | `actor['login']` | STRING | +| repo_name | `repo['name']` | STRING | +| payload_action | `payload['action']` | STRING | +| actor_id | `actor['id']` | BIGINT | + +The default pair is `actor_login,actor_id`, covering one string and one numeric +path. Use `--keys` to select any subset of the four paths. + +Each query has native VARIANT and explicit CAST forms. SQL cache and query cache +are disabled. GROUP BY correctness uses bidirectional EXCEPT between complete +native and CAST group results before timing compact aggregate checksums. JOIN +checks count and ID sum and uses one dimension row per key to avoid many-to-many +output explosion. + +| Operation | What it measures | +| --- | --- | +| `group` | GROUP BY on the key, reduced to a compact checksum | +| `order_topn` | `ORDER BY key NULLS FIRST, id LIMIT 1000`: a TopN, not a full sort | +| `order_full` | `row_number() OVER (ORDER BY key NULLS FIRST, id)` over every row: a full sort whose `sum(id * rn)` checksum compares the complete native and CAST orders | +| `join_broadcast`, `join_shuffle` | equality join against the dimension table | + +Variant join keys get no runtime filter while CAST keys do. Each join therefore +has three modes: `native`, `cast`, and `cast_no_rf` (the CAST join with +`runtime_filter_mode='OFF'`). Compare `native` with `cast_no_rf` for the key +comparison cost and `cast` with `cast_no_rf` for the runtime filter benefit. + +A join against an empty or stale dimension table would still make native and +CAST agree. Before timing a join, the query phase checks with bidirectional +EXCEPT that the dimension table equals the current grouping of `github_events` +and is not empty; rerun the load phase (or `VARIANT_BENCH_PHASE=prepare`) if it +fails. + +Mixed-type semantics belong in `variant_p0/test_variant_relational_corners.groovy`. +They are deliberately excluded from these performance pairs: CAST may merge +strings with numbers or round integers above 2^53. Matching CAST results alone +cannot prove correct native mixed-type behavior. + +Build and start this worktree's FE/BE using the repository environment workflow. +Performance requires `BUILD_TYPE=RELEASE` and `./build.sh --be --fe`; do not pass +an explicit `-j`. Synchronize worktree ports after building. The runner requires +Python 3.11+, Linux `/proc`, and the local `output/` runtime. Use a dedicated +database/config because `load.groovy` recreates `github_events`. + +Run from the worktree root, substituting the verified regression config: + +```bash +python3 regression-test/suites/variant_p2/run_relational_benchmark.py load \ + --conf tmp/regression-conf.auto.groovy --stream-load \ + --output tmp/relational-load + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --warmups 2 --repeats 7 \ + --output tmp/relational-query + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --keys actor_login \ + --warmups 0 --repeats 3 --spill --output tmp/relational-spill +``` + +Choose eight distinct physical cores after inspecting host topology and load. +The load phase does not change affinity and rejects an already eight-core-bound +FE or BE process. Doris may independently pin a few service threads. The query +phase binds every FE and BE thread to the same eight cores, checks affinity +during the run, and restores previous masks afterward. This restricts the +database servers; the regression client runs outside that mask. CPU affinity +does not reserve cores against unrelated host processes. + +The runner verifies the running BE matches the installed Release binary. Each +new evidence directory contains revision/binary/affinity metadata, host load, +regression logs, query plans, every warmup/measured latency and result SHA-256, +and median/min/max summaries. Latency includes client round-trip and result +consumption. The starting mode rotates each round. Interpret forced-spill +measurements separately from ordinary queries. A completed repeat run checks +repeatability, not long-duration soak, concurrent ingestion, or crash recovery. + +The query suite can also run directly through `run-regression-test.sh` using +`VARIANT_BENCH_PHASE=prepare|query` (required: without it an ordinary `variant_p2` +run skips the suite), `VARIANT_BENCH_ROWS`, `VARIANT_BENCH_KEYS`, +`VARIANT_BENCH_REPEATS`, `VARIANT_BENCH_WARMUPS`, `VARIANT_BENCH_RESULTS`, and +`VARIANT_BENCH_SPILL`. Direct runs do not enforce Release or CPU affinity and +must not be reported as an eight-core performance result. diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 8cb0bc19255f30..1e63dd91a0e76e 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -18,7 +18,11 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit suite("load_p2", "variant_type,p2"){ - boolean use_stream_load = false + def env = System.getenv() + boolean use_stream_load = env + .getOrDefault("VARIANT_P2_USE_STREAM_LOAD", "false").toBoolean() + def resumeFiles = env.getOrDefault("VARIANT_P2_RESUME_FILES", "") + .split(",").findAll { !it.isEmpty() } def load_json_data = {table_name, file_name -> // load the json data streamLoad { @@ -30,6 +34,8 @@ suite("load_p2", "variant_type,p2"){ set 'max_filter_ratio', '0.1' file file_name // import json file time 10000 // limit inflight 10s + timeout 60000 + retryIfHttpError true // if declared a check callback, the default check condition will ignore. // So you must check all condition @@ -124,37 +130,41 @@ suite("load_p2", "variant_type,p2"){ } } } - create_table.call(table_name) - List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - // 2015 - def year = "2015" - def monthPrefix = "0" - def dayPrefix = "0" - log.info("current year: ${year}") - for (int i = 1; i <= 3; i++) { - def month = i < 10 ? monthPrefix + i.toString() : i.toString() - log.info("current month: ${month}") - for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { - def day = j < 10 ? dayPrefix + j.toString() : j.toString() - log.info("current day: ${day}") - for (int z = 1; z < 24; z++) { - def hour = z.toString() - log.info("current hour: ${hour}") - def fileName = year + "-" + month + "-" + day + "-" + hour + ".json" - log.info("cuurent fileName: ${fileName}") - if (use_stream_load) { - def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - load_json_data.call(table_name, fileUrl) - } as Runnable) - } else { - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - s3load_paral_wait.call(table_name, "JSON", "regression/github_events_dataset/${fileName}", 3) - } as Runnable) + def submitFile = { fileName -> + if (use_stream_load) { + def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + load_json_data.call(table_name, fileUrl) + } as Runnable) + } else { + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + s3load_paral_wait.call(table_name, "JSON", + "regression/github_events_dataset/${fileName}", 3) + } as Runnable) + } + } + if (resumeFiles) { + assertTrue(use_stream_load, "Resume requires VARIANT_P2_USE_STREAM_LOAD=true") + resumeFiles.each { submitFile.call(it) } + } else { + create_table.call(table_name) + List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + def year = "2015" + def monthPrefix = "0" + def dayPrefix = "0" + log.info("current year: ${year}") + for (int i = 1; i <= 3; i++) { + def month = i < 10 ? monthPrefix + i.toString() : i.toString() + log.info("current month: ${month}") + for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { + def day = j < 10 ? dayPrefix + j.toString() : j.toString() + log.info("current day: ${day}") + for (int z = 1; z < 24; z++) { + def fileName = year + "-" + month + "-" + day + "-" + z + ".json" + log.info("current fileName: ${fileName}") + submitFile.call(fileName) } } } diff --git a/regression-test/suites/variant_p2/relational_performance.groovy b/regression-test/suites/variant_p2/relational_performance.groovy new file mode 100644 index 00000000000000..2dcfc19a2fab1e --- /dev/null +++ b/regression-test/suites/variant_p2/relational_performance.groovy @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import java.security.MessageDigest + +suite("variant_relational_performance", "p2,nonConcurrent") { + def env = System.getenv() + // This benchmark is driven by run_relational_benchmark.py. A plain variant_p2 run has no + // prepared dimension tables, so the suite only runs when a phase is set explicitly. + def phase = env.get("VARIANT_BENCH_PHASE") + if (phase == null) { + log.info("Skip variant_relational_performance: VARIANT_BENCH_PHASE is not set") + return + } + long expectedRows = env.getOrDefault("VARIANT_BENCH_ROWS", "44273863").toLong() + int repeats = env.getOrDefault("VARIANT_BENCH_REPEATS", "7").toInteger() + int warmups = env.getOrDefault("VARIANT_BENCH_WARMUPS", "2").toInteger() + if (!(phase in ["prepare", "query"]) || expectedRows < 1 || repeats < 1 || warmups < 0) { + throw new IllegalArgumentException("Invalid VARIANT_BENCH configuration") + } + // These paths come directly from variant_p2/sql and the GitHub Events schema. + def keys = [ + actor_login: [column: "actor", path: "login", type: "STRING"], + repo_name: [column: "repo", path: "name", type: "STRING"], + payload_action: [column: "payload", path: "action", type: "STRING"], + actor_id: [column: "actor", path: "id", type: "BIGINT"] + ] + def requestedKeys = env.getOrDefault("VARIANT_BENCH_KEYS", "actor_login,actor_id").split(",") as Set + if (!keys.keySet().containsAll(requestedKeys)) { + throw new IllegalArgumentException("Unknown VARIANT_BENCH_KEYS: ${requestedKeys - keys.keySet()}") + } + keys = keys.findAll { key, ignored -> requestedKeys.contains(key) } + + def actualRows = (sql("SELECT count(*) FROM github_events"))[0][0].toString().toLong() + assertEquals(expectedRows, actualRows) + if (phase == "prepare") { + sql "SET default_variant_max_subcolumns_count = 0" + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + sql "DROP TABLE IF EXISTS variant_relational_dim_${key}" + sql """CREATE TABLE variant_relational_dim_${key} (id BIGINT, k VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_dim_${key} + SELECT min(id), ${nativeKey} FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${nativeKey}""" + } + return + } + + sql "SET parallel_pipeline_task_num = 8" + sql "SET enable_sql_cache = false" + sql "SET enable_query_cache = false" + if (env.getOrDefault("VARIANT_BENCH_SPILL", "false").toBoolean()) { + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + } + def output = new File(env.getOrDefault("VARIANT_BENCH_RESULTS", + "tmp/variant-relational-results.jsonl")) + output.parentFile.mkdirs() + def record = { event -> output.append(JsonOutput.toJson(event) + "\n") } + def fingerprint = { result -> + MessageDigest.getInstance("SHA-256").digest(JsonOutput.toJson( + result.collect { row -> row.collect { value -> value == null ? null : value.toString() } } + ).getBytes("UTF-8")).encodeHex().toString() + } + def query = { statement -> + long start = System.nanoTime() + def result = sql(statement) + [ms: (System.nanoTime() - start) / 1e6, hash: fingerprint(result), resultRows: result.size()] + } + record([event: "start", rows: actualRows, repeats: repeats, warmups: warmups, + parallelPipelineTasks: 8, keys: keys.keySet(), cpus: env.get("VARIANT_BENCH_CPUS"), + spill: env.get("VARIANT_BENCH_SPILL"), time: new Date().toString()]) + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + def castKey = "CAST(${nativeKey} AS ${spec.type})" + def nativeGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${nativeKey}""" + def castGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${castKey}""" + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${nativeGroups}) EXCEPT (${castGroups})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${castGroups}) EXCEPT (${nativeGroups})) difference"""))[0][0].toString().toInteger()) + record([event: "full_group_correctness", key: key]) + + // ORDER BY ... LIMIT is a TopN and never sorts the whole input. The window forces a + // full sort of every row, and its order-sensitive checksum compares the complete + // native order with the CAST order instead of the first rows only. + def fullSort = { orderKey -> """SELECT count(*), sum(CAST(id AS LARGEINT) * rn) FROM ( + SELECT id, row_number() OVER (ORDER BY ${orderKey} NULLS FIRST, id) rn + FROM github_events) ranked""" } + def queries = [ + group: [ + native: "SELECT count(*), sum(n*n), sum(first_id) FROM (${nativeGroups}) g", + cast: "SELECT count(*), sum(n*n), sum(first_id) FROM (${castGroups}) g" + ], + order_topn: [ + native: "SELECT id FROM github_events ORDER BY ${nativeKey} NULLS FIRST, id LIMIT 1000", + cast: "SELECT id FROM github_events ORDER BY ${castKey} NULLS FIRST, id LIMIT 1000" + ], + order_full: [native: fullSort(nativeKey), cast: fullSort(castKey)] + ] + + // A join against an empty or stale dimension still makes native and CAST agree, so + // prove the dimension is the current grouping of the source before timing a join. + def dimension = "SELECT id, CAST(k AS ${spec.type}) k FROM variant_relational_dim_${key}" + def expectedDimension = """SELECT min(id) id, ${castKey} k FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${castKey}""" + def dimensionRows = (sql("SELECT count(*) FROM variant_relational_dim_${key}"))[0][0] + .toString().toLong() + assertTrue(dimensionRows > 0) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${dimension}) EXCEPT (${expectedDimension})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${expectedDimension}) EXCEPT (${dimension})) difference"""))[0][0].toString().toInteger()) + record([event: "dimension_verified", key: key, rows: dimensionRows]) + + // Variant keys get no runtime filter while the CAST keys do, so cast_no_rf isolates the + // key comparison cost from the runtime filter benefit. + def leftKey = "l.${spec.column}['${spec.path}']" + def castPredicate = "CAST(${leftKey} AS ${spec.type}) = CAST(r.k AS ${spec.type})" + ["broadcast", "shuffle"].each { distribution -> + def join = { hint, predicate -> """SELECT ${hint} count(*), sum(l.id) FROM github_events l + JOIN [${distribution}] variant_relational_dim_${key} r ON ${predicate}""" } + queries["join_${distribution}".toString()] = [ + native: join("", "${leftKey} = r.k"), + cast: join("", castPredicate), + cast_no_rf: join("/*+ SET_VAR(runtime_filter_mode='OFF') */", castPredicate) + ] + } + + queries.each { operation, statements -> + def oracle = query(statements.cast).hash + statements.each { mode, statement -> + record([event: "plan", key: key, operation: operation, mode: mode, + plan: sql("EXPLAIN ${statement}")]) + } + def modes = statements.keySet() as List + for (int round = -warmups; round < repeats; ++round) { + // Rotate the starting mode so no mode always runs first or last. + Collections.rotate(modes, 1) + modes.each { mode -> + def sample = query(statements[mode]) + assertEquals(oracle, sample.hash) + record(sample + [event: "sample", key: key, operation: operation, + mode: mode, round: round, sql: statements[mode]]) + } + } + } + } + record([event: "complete", time: new Date().toString()]) +} diff --git a/regression-test/suites/variant_p2/run_relational_benchmark.py b/regression-test/suites/variant_p2/run_relational_benchmark.py new file mode 100644 index 00000000000000..72e9a19660dd89 --- /dev/null +++ b/regression-test/suites/variant_p2/run_relational_benchmark.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run the regression suite with unrestricted load and verified 8-core queries.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import statistics +import subprocess +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("load", "query")) + parser.add_argument("--conf", required=True) + parser.add_argument("--cpus", help="Eight comma-separated physical CPU IDs (query only)") + parser.add_argument("--rows", type=int, default=44_273_863) + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--keys", default="actor_login,actor_id") + parser.add_argument("--stream-load", action="store_true", + help="Load public variant_p2 files over HTTP instead of authenticated S3") + parser.add_argument("--resume-files", default="", + help="Comma-separated public variant_p2 files to append after an interrupted load") + parser.add_argument("--spill", action="store_true", help="Separate forced-spill correctness/stability run") + parser.add_argument("--output", required=True, help="New evidence directory") + args = parser.parse_args() + if args.phase == "query" and (args.stream_load or args.resume_files): + parser.error("--stream-load and --resume-files apply only to the load phase") + if args.resume_files and not args.stream_load: + parser.error("--resume-files requires --stream-load") + # Job cancellation sends SIGTERM. Unwind through the finally block below so the regression + # child is stopped and FE/BE thread affinity is restored. + signal.signal(signal.SIGTERM, lambda signum, _: sys.exit(128 + signum)) + repo = Path(__file__).resolve().parents[3] + evidence = Path(args.output).resolve() + evidence.mkdir(parents=True, exist_ok=False) + processes = {} + original = {} + for name in ("be", "fe"): + pid = int((repo / f"output/{name}/bin/{name}.pid").read_text()) + command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode() + if str(repo) not in command: + raise RuntimeError(f"Refusing to change unrelated {name} PID {pid}") + processes[name] = pid + original[pid] = {int(t.name): os.sched_getaffinity(int(t.name)) + for t in Path(f"/proc/{pid}/task").iterdir()} + # The regression client must query the FE whose threads are pinned and fingerprinted here. + conf_port = re.search(r"jdbc:mysql://[^:/]+:(\d+)", Path(args.conf).read_text()) + fe_port = re.search(r"^\s*query_port\s*=\s*(\d+)", + (repo / "output/fe/conf/fe.conf").read_text(), re.MULTILINE) + if conf_port is None or int(conf_port.group(1)) != int(fe_port.group(1) if fe_port else 9030): + raise RuntimeError("--conf does not point at this worktree's FE query port") + cpus = None + if args.phase == "query": + cpus = {int(cpu) for cpu in (args.cpus or "").split(",") if cpu} + topology = {tuple((Path(f"/sys/devices/system/cpu/cpu{cpu}/topology") / item) + .read_text().strip() for item in ("physical_package_id", "core_id")) + for cpu in cpus} + if len(cpus) != 8 or len(topology) != 8: + raise ValueError("Select exactly eight distinct physical cores") + cache = (repo / "be/build_RELEASE/CMakeCache.txt").read_text() + if "CMAKE_BUILD_TYPE:STRING=RELEASE" not in cache.upper(): + raise RuntimeError("A Release build is required for performance measurements") + binary = repo / "output/be/lib/doris_be" + running = Path(f"/proc/{processes['be']}/exe") + if not os.path.samefile(binary, running): + raise RuntimeError("Running BE differs from the worktree output binary") + release_binary = repo / "be/build_RELEASE/src/service/doris_be" + with release_binary.open("rb") as built, binary.open("rb") as installed: + if hashlib.file_digest(built, "sha256").digest() != hashlib.file_digest(installed, "sha256").digest(): + raise RuntimeError("Installed BE does not match the Release build") + elif args.cpus: + raise ValueError("Do not bind CPUs during ingestion") + else: + for pid in processes.values(): + if len(os.sched_getaffinity(pid)) <= 8: + raise RuntimeError("Load requires each FE/BE process to be unrestricted beyond eight CPUs") + manifest = dict(vars(args), processes=processes, checkout=str(repo), + head=subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip(), + started=time.time(), original_affinity={str(p): {str(t): sorted(m) for t, m in ts.items()} + for p, ts in original.items()}) + manifest["harness_sha256"] = { + name: hashlib.sha256((Path(__file__).parent / name).read_bytes()).hexdigest() + for name in ("load.groovy", "relational_performance.groovy", Path(__file__).name) + } + manifest["dirty_checkout"] = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], cwd=repo, text=True).strip()) + # Hash what the recorded processes run, not only what the checkout contains. + with Path(f"/proc/{processes['be']}/exe").open("rb") as binary: + manifest["be_sha256"] = hashlib.file_digest(binary, "sha256").hexdigest() + with (repo / "output/fe/lib/doris-fe.jar").open("rb") as jar: + manifest["fe_jar_sha256"] = hashlib.file_digest(jar, "sha256").hexdigest() + (evidence / "manifest.json").write_text(json.dumps(manifest, indent=2)) + environment = dict(os.environ, VARIANT_BENCH_PHASE=args.phase, + VARIANT_BENCH_ROWS=str(args.rows), VARIANT_BENCH_REPEATS=str(args.repeats), + VARIANT_BENCH_WARMUPS=str(args.warmups), + VARIANT_BENCH_KEYS=args.keys, + VARIANT_BENCH_RESULTS=str(evidence / "samples.jsonl"), + VARIANT_BENCH_SPILL=str(args.spill).lower(), + VARIANT_P2_USE_STREAM_LOAD=str(args.stream_load).lower(), + VARIANT_P2_RESUME_FILES=args.resume_files, + VARIANT_BENCH_CPUS=args.cpus or "unrestricted") + for key in list(environment): + if key.lower() in ("http_proxy", "https_proxy", "all_proxy"): + del environment[key] + environment["NO_PROXY"] = environment["no_proxy"] = "127.0.0.1,localhost" + child = None + try: + if cpus: + # Re-enumerate until every thread has the mask. New threads inherit + # the creator's mask; the monitor below rejects any subsequent drift. + for pid in processes.values(): + for _ in range(10): + tids = [int(t.name) for t in Path(f"/proc/{pid}/task").iterdir()] + for tid in tids: + try: + os.sched_setaffinity(tid, cpus) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(t.name)) == cpus + for t in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not establish CPU affinity") + suites = (["load_p2", "variant_relational_performance"] + if args.phase == "load" else ["variant_relational_performance"]) + with (evidence / "host-load.jsonl").open("w") as host: + for suite in suites: + suite_environment = environment.copy() + if args.phase == "load" and suite == "variant_relational_performance": + suite_environment["VARIANT_BENCH_PHASE"] = "prepare" + log_name = ({"load_p2": "load.log"}.get(suite, "prepare.log") + if args.phase == "load" else "regression.log") + command = [str(repo / "run-regression-test.sh"), "--conf", args.conf, "--run", + "-d", "variant_p2", "-s", suite] + with (evidence / log_name).open("w") as log: + child = subprocess.Popen(command, cwd=repo, env=suite_environment, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + while child.poll() is None: + host.write(json.dumps({"time": time.time(), + "loadavg": Path("/proc/loadavg").read_text().strip()}) + "\n") + host.flush() + if cpus: + for pid in processes.values(): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + if os.sched_getaffinity(int(thread.name)) != cpus: + raise RuntimeError(f"CPU affinity drift: {thread}") + except ProcessLookupError: + pass + time.sleep(1) + if child.returncode: + raise RuntimeError(f"Regression failed: inspect {evidence / log_name}") + child = None + finally: + if child is not None and child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + child.wait() + if cpus: + for pid, threads in original.items(): + for _ in range(10): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + os.sched_setaffinity(int(thread.name), threads.get(int(thread.name), threads[pid])) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(thread.name)) == + threads.get(int(thread.name), threads[pid]) + for thread in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not restore CPU affinity") + if args.phase == "query": + events = [json.loads(line) for line in (evidence / "samples.jsonl").read_text().splitlines()] + if not events or events[-1]["event"] != "complete": + raise RuntimeError("Missing completion record") + groups = {} + for event in events: + if event["event"] == "sample" and event["round"] >= 0: + groups.setdefault((event["key"], event["operation"], event["mode"]), []).append(event["ms"]) + summary = [{"key": key, "operation": operation, "mode": mode, "samples_ms": values, + "median_ms": statistics.median(values), "min_ms": min(values), "max_ms": max(values)} + for (key, operation, mode), values in groups.items()] + (evidence / "summary.json").write_text(json.dumps(summary, indent=2)) + print(evidence) + + +if __name__ == "__main__": + main()