diff --git a/be/src/core/column/variant_v2/column_variant_v2.cpp b/be/src/core/column/variant_v2/column_variant_v2.cpp index 7d7317d1e0ca7c..70fc0d25c49619 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.cpp +++ b/be/src/core/column/variant_v2/column_variant_v2.cpp @@ -45,6 +45,7 @@ #include "core/value/variant/variant_canonical.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/sort/sort_block.h" namespace doris { namespace { @@ -689,6 +690,33 @@ const IColumn& ColumnVariantV2::typed_column() const { return *_typed; } +void ColumnVariantV2::validate_typed_rows() const { + DORIS_CHECK(_typed != nullptr) << "validate_typed_rows requires ColumnVariantV2 typed state"; + switch (_typed_type->get_primitive_type()) { + // Every value of these types maps to a Variant scalar, so there is nothing to reject and no + // reason to format each LARGEINT or IP address only to discard the text. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_IPV4: + case TYPE_IPV6: + return; + default: + break; + } + // with_variant_typed_scalar() is the one definition of what Variant can encode: it throws for + // an invalid date, a string that is not UTF-8 or a decimal outside its width. A type missing + // from the list above is validated, which is the safe side. + visit_typed_scalar_column(assert_cast(*_typed), + _typed_type->get_primitive_type(), _typed_type->get_scale(), 0, + _typed->size(), [](size_t, const VariantScalarRef&) {}); +} + const DataTypePtr& ColumnVariantV2::typed_type() const { DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state"; return _typed_type; @@ -1285,6 +1313,132 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row) const { return {.metadata = {.data = metadata.data, .size = metadata.size}, .value = value}; } +std::pair ColumnVariantV2::_typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const { + if (!is_typed() || !right.is_typed() || + !(_typed_type == right._typed_type || _typed_type->equals(*right._typed_type))) { + return {nullptr, nullptr}; + } + // Admission is a whitelist: a typed type takes the native path only after its native order has + // been checked against the canonical Variant order (scalar_compare in variant_canonical.cpp). + // The premises shared by every admitted type: + // - both sides have the same typed type including its parameters, so DECIMAL scales and + // DATETIME precisions match and native values are directly comparable; + // - every non-null typed row is a value Variant can encode (the rule and who enforces it + // are at create_typed()), so the native order does not rank a value the canonical path + // would reject, such as an invalid date or a string that is not UTF-8; + // - a typed NULL is the Variant null, which the callers order first, as the canonical order + // does. + // A type that is not listed falls back to the canonical comparison, which is always correct. + switch (_typed_type->get_primitive_type()) { + // Exact numerics and temporal values: both orders are the numeric/chronological order. + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DATE: + case TYPE_DATEV2: + case TYPE_DATETIME: + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMPTZ: + // Strings: both orders compare the bytes as unsigned. + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + break; + // Canonical order puts every NaN last and equal to each other, and -0 equal to +0. compare_at() + // of a floating point column orders the same way; its compare_internal() uses < and >, which + // leave NaN unordered. + case TYPE_FLOAT: + case TYPE_DOUBLE: + if (!allow_floating) { + return {nullptr, nullptr}; + } + break; + // Not admitted: LARGEINT magnitudes above 10^38 - 1 are Variant strings, and IPv4/IPv6 are + // Variant strings whose byte order is not the address order. + default: + return {nullptr, nullptr}; + } + return {&assert_cast(typed_column()), + &assert_cast(right.typed_column())}; +} + +int ColumnVariantV2::compare_at(size_t n, size_t m, const IColumn& rhs, + int nan_direction_hint) const { + const auto& right = assert_cast(rhs); + DCHECK_LT(n, size()); + DCHECK_LT(m, right.size()); + + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, true); + left_nullable != nullptr) { + if (left_nullable->is_null_at(n)) { + return right_nullable->is_null_at(m) ? 0 : -1; + } + if (right_nullable->is_null_at(m)) { + return 1; + } + return left_nullable->get_nested_column().compare_at( + n, m, right_nullable->get_nested_column(), nan_direction_hint); + } + + // A column is in exactly one of three states. Encoded rows are canonical Variant bytes that + // get_value_ref() reads in place. A shredded column has no row bytes until its state + // materializes them; get_value_ref() asks the state for them, which builds and caches them on + // the first call. A typed row is a scalar, compared below without being encoded: a sort compares + // O(n log n) times, so nothing here may allocate or encode per comparison. + if (!is_typed() && !right.is_typed()) { + return canonical_compare(get_value_ref(n), right.get_value_ref(m)); + } + if (!is_typed()) { + // Keep the typed side on the left; canonical_compare() is antisymmetric. + return -right.compare_at(m, n, *this, nan_direction_hint); + } + // The typed fast path declined (different typed types, a type outside its whitelist, or an + // encoded/shredded right side). A typed null row is the canonical null scalar, which sorts + // first - the same order the typed fast path applies. + int result = 0; + const auto& left_nullable = + assert_cast(typed_column()); + visit_typed_scalar_column( + left_nullable, _typed_type->get_primitive_type(), _typed_type->get_scale(), n, n + 1, + [&](size_t, const VariantScalarRef& left_scalar) { + if (!right.is_typed()) { + result = canonical_compare(left_scalar, right.get_value_ref(m)); + return; + } + visit_typed_scalar_column( + assert_cast( + right.typed_column()), + right._typed_type->get_primitive_type(), right._typed_type->get_scale(), m, + m + 1, [&](size_t, const VariantScalarRef& right_scalar) { + result = canonical_compare(left_scalar, right_scalar); + }); + }); + return result; +} + +void ColumnVariantV2::compare_internal(size_t rhs_row_id, const IColumn& rhs, + int nan_direction_hint, int direction, + std::vector& cmp_res, + uint8_t* __restrict filter) const { + const auto& right = assert_cast(rhs); + // ColumnVector::compare_internal does not provide Variant's canonical NaN ordering, so floating + // point typed columns take the canonical path below. + if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, false); + left_nullable != nullptr) { + left_nullable->compare_internal(rhs_row_id, *right_nullable, -1, direction, cmp_res, + filter); + return; + } + IColumn::compare_internal(rhs_row_id, rhs, nan_direction_hint, direction, cmp_res, filter); +} + Field ColumnVariantV2::operator[](size_t row) const { Field result; get(row, result); @@ -1662,6 +1816,29 @@ void ColumnVariantV2::pop_back(size_t length) { _check_invariants(); } +void ColumnVariantV2::erase(size_t start, size_t length) { + DORIS_CHECK_LE(start, size()) << "erase start exceeds the column size"; + DORIS_CHECK_LE(length, size() - start) << "erase range exceeds the column size"; + if (length == 0) { + return; + } + if (_shredded) { + ensure_encoded(); + } + if (_typed) { + mutate_subcolumn(_typed); + _typed->erase(start, length); + _check_invariants(); + return; + } + require_exclusive(_meta_ids, "metadata ids"); + require_exclusive(_values, "values"); + // The metadata dictionary keeps blobs that no remaining row references; ids stay valid. + _values->erase(start, length); + _meta_ids->erase(start, length); + _check_invariants(); +} + StringRef ColumnVariantV2::get_data_at(size_t) const { throw_unsupported("get_data_at"); } @@ -2139,8 +2316,32 @@ void ColumnVariantV2::resize(size_t new_size) { } } -void ColumnVariantV2::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const { - throw_unsupported("get_permutation"); +void ColumnVariantV2::get_permutation(bool reverse, size_t limit, int nan_direction_hint, + HybridSorter& sorter, Permutation& result) const { + const size_t row_count = size(); + result.resize(row_count); + for (size_t row = 0; row < row_count; ++row) { + result[row] = row; + } + + const auto less = [&](size_t left, size_t right) { + const int comparison = compare_at(left, right, *this, nan_direction_hint); + return reverse ? comparison > 0 : comparison < 0; + }; + // std::partial_sort only pays off when limit << row_count, the same threshold ColumnVector uses. + if (static_cast(limit) > static_cast(row_count) / 8.0) { + limit = 0; + } + if (limit != 0 && limit < row_count) { + std::partial_sort(result.begin(), result.begin() + limit, result.end(), less); + } else { + sorter.sort(result.begin(), result.end(), less); + } +} + +void ColumnVariantV2::sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const { + sorter->sort_column(*this, flags, perms, range, last_column); } void ColumnVariantV2::replace_column_data(const IColumn&, size_t, size_t) { diff --git a/be/src/core/column/variant_v2/column_variant_v2.h b/be/src/core/column/variant_v2/column_variant_v2.h index affea6a7e34b7a..d096abd23fa8d8 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.h +++ b/be/src/core/column/variant_v2/column_variant_v2.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "common/exception.h" #include "core/assert_cast.h" @@ -179,7 +180,21 @@ class ColumnVariantV2 final : public COWHelper { // The input must be an exact, non-Const ColumnNullable whose nested column matches the // non-nullable supported scalar type. + // + // Every row that is non-null in the typed column's own null map must be a value Variant can + // encode. That null map is the only mask every consumer honours: hashing, for one, maps a row + // to a scalar before it looks at an outer null map. Nothing is checked here, because validity + // is enforced where a row becomes a scalar (with_variant_typed_scalar() throws), which covers + // hashing, serialization, encoding and the canonical comparison. The native comparison paths + // do not go through that mapping and rely on the rule instead: + // - producers that decode stored or transported Variant data build valid rows; an external + // Parquet file is trusted to follow its specification, and a violation surfaces as an + // error when the row is hashed or encoded; + // - CAST from a scalar takes arbitrary values, so it folds its null map into the typed one + // and calls validate_typed_rows(). static MutablePtr create_typed(ColumnPtr column, DataTypePtr scalar_type); + // Throws unless every non-null typed row is a value Variant can encode. + void validate_typed_rows() const; static MutablePtr create_shredded(std::shared_ptr state); bool is_typed() const noexcept { return _typed != nullptr; } @@ -224,6 +239,11 @@ class ColumnVariantV2 final : public COWHelper { // including insert, clear, COW mutation, or future row transformations, may invalidate it. VariantRef get_value_ref(size_t row) const; + int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override; + void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint, + int direction, std::vector& cmp_res, + uint8_t* __restrict filter) const override; + Field operator[](size_t row) const override; void get(size_t row, Field& result) const override; void insert(const Field& field) override; @@ -235,6 +255,8 @@ class ColumnVariantV2 final : public COWHelper { void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, const uint32_t* indices_end) override; void pop_back(size_t length) override; + // The analytic sink erases the rows it has consumed from its partition and order key columns. + void erase(size_t start, size_t length) override; StringRef get_data_at(size_t row) const override; void insert_data(const char* pos, size_t length) override; @@ -272,6 +294,8 @@ class ColumnVariantV2 final : public COWHelper { void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter, Permutation& result) const override; + void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms, + EqualRange& range, bool last_column) const override; void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override; private: @@ -289,6 +313,9 @@ class ColumnVariantV2 final : public COWHelper { void _adopt_state_from(ColumnVariantV2& replacement); void _detach_metadata_for_write(); void _check_invariants() const; + std::pair _typed_ordering_operands( + const ColumnVariantV2& right, bool allow_floating) const; + static void mutate_subcolumn(IColumn::WrappedPtr& subcolumn) { static_cast(subcolumn) = std::move(*static_cast(subcolumn)).mutate(); diff --git a/be/src/core/data_type/data_type_variant_v2.cpp b/be/src/core/data_type/data_type_variant_v2.cpp index 92b0fb4cbd209b..456bc3884483d9 100644 --- a/be/src/core/data_type/data_type_variant_v2.cpp +++ b/be/src/core/data_type/data_type_variant_v2.cpp @@ -48,10 +48,10 @@ MutableColumnPtr DataTypeVariantV2::create_column() const { } bool DataTypeVariantV2::equals(const IDataType& rhs) const { - const auto* rhs_type = typeid_cast(&rhs); - return rhs_type != nullptr && - _max_subcolumns_count == rhs_type->variant_max_subcolumns_count() && - _enable_doc_mode == rhs_type->enable_doc_mode(); + // The max subcolumns count and doc mode only guide how storage shreds a Variant column. + // Execution compares, hashes and serializes Variant values canonically without them, so two + // Variant types with different properties are the same execution type (e.g. hash join keys). + return typeid_cast(&rhs) != nullptr; } int64_t DataTypeVariantV2::get_uncompressed_serialized_bytes(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index e2ecf2cb5e0536..9606b880f7b8c2 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -228,7 +228,8 @@ Status DataTypeArraySerDe::serialize_one_cell_to_hive_text( } Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& data_column = assert_cast(from_column); const auto& offsets = data_column.get_offsets(); @@ -241,7 +242,7 @@ Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, } for (size_t i = start; i < end; ++i) { - RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer)); + RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer, options)); } if (!writer.writeEndArray()) { return Status::InternalError("writeEndArray failed"); diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index eaf3644efc6879..31378a61081ede 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -122,7 +122,8 @@ class DataTypeArraySerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index acd409be1af91e..c783245130c2b5 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -1239,8 +1239,8 @@ void DataTypeDecimalSerDe::to_string_batch(const IColumn& column, ColumnStrin template Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb"); } else { @@ -1254,8 +1254,9 @@ Status DataTypeDecimalSerDe::serialize_column_to_jsonb(const IColumn& from_co } template -Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { if constexpr (T == TYPE_DECIMALV2) { return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb_vector"); } else { @@ -1263,6 +1264,10 @@ Status DataTypeDecimalSerDe::serialize_column_to_jsonb_vector(const IColumn& JsonbWriter writer; const auto& data = assert_cast&>(from_column).get_data(); for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); if (!writer.writeDecimal(data[i], precision, scale)) { return Status::InvalidArgument( diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 0b01da4bc32230..fc6263b79c8144 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -92,10 +92,12 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; - Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp index c728532eafb977..6055bb3f19572d 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp @@ -342,7 +342,8 @@ void convert_jsonb_to_rapidjson(const JsonbValue& val, rapidjson::Value& target, } Status DataTypeJsonbSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& jsonb_binary = assert_cast(from_column).get_data_at(row_num); const JsonbDocument* doc = nullptr; RETURN_IF_ERROR( diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.h b/be/src/core/data_type_serde/data_type_jsonb_serde.h index 3a243a1c1a8187..4a50aefa5d09e1 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.h +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.h @@ -75,7 +75,8 @@ class DataTypeJsonbSerDe : public DataTypeStringSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index 1feec24aa5841b..b25011de53e022 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -769,7 +769,8 @@ Status DataTypeMapSerDe::from_string_strict_mode(StringRef& str, IColumn& column } Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& map_column = assert_cast(from_column); const ColumnArray::Offsets64& offsets = map_column.get_offsets(); @@ -803,7 +804,7 @@ Status DataTypeMapSerDe::serialize_column_to_jsonb(const IColumn& from_column, i return Status::InternalError("writeKey failed : {}", key_str.to_string()); } // write value - RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer)); + RETURN_IF_ERROR(value_serde->serialize_column_to_jsonb(values_column, i, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_map_serde.h b/be/src/core/data_type_serde/data_type_map_serde.h index 1404963ecf3e10..81ae4ae4724030 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.h +++ b/be/src/core/data_type_serde/data_type_map_serde.h @@ -118,7 +118,8 @@ class DataTypeMapSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {key_serde, value_serde}; diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index de8c896f903848..55f140217abeba 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -125,14 +125,15 @@ Status DataTypeNullableSerDe::serialize_one_cell_to_hive_text( } Status DataTypeNullableSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& col_null = assert_cast(from_column); if (col_null.is_null_at(row_num)) { writer.writeNull(); } else { RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(col_null.get_nested_column(), - row_num, writer)); + row_num, writer, options)); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index 8be1ece1e61a5f..3e1f0188ef2b19 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -125,7 +125,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe { } Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 029a5875d2b812..8ea6c7bfe6b1f7 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -1169,8 +1169,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) { template Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } @@ -1183,8 +1183,9 @@ Status DataTypeNumberSerDe::serialize_column_to_jsonb(const IColumn& from_col } template -Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { if constexpr (!can_write_to_jsonb_from_number()) { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } @@ -1193,6 +1194,10 @@ Status DataTypeNumberSerDe::serialize_column_to_jsonb_vector(const IColumn& f const auto& data = assert_cast(from_column).get_data(); const auto scale = get_scale(); for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); if (!write_to_jsonb_from_number(data[i], writer, scale)) { return Status::InvalidArgument( diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 92828d74a163b8..571888ff34bfba 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -88,10 +88,12 @@ class DataTypeNumberSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; - Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index bd2a09b6b6e68c..b78e5615aa7f3c 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -144,13 +144,18 @@ Status DataTypeSerDe::default_from_string(StringRef& str, IColumn& column) const return deserialize_one_cell_from_json(column, slice, options); } -Status DataTypeSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const { +Status DataTypeSerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { const auto size = from_column.size(); JsonbWriter writer; for (int i = 0; i < size; i++) { + if (source_null_map && source_null_map[i]) { + to_column.insert_default(); + continue; + } writer.reset(); - RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer)); + RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer, options)); to_column.insert_data(writer.getOutput()->getBuffer(), writer.getOutput()->getSize()); } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index b4cb8936aa369e..e44bedc1a4911a 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -439,12 +439,15 @@ class DataTypeSerDe { } virtual Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name()); } - virtual Status serialize_column_to_jsonb_vector(const IColumn& from_column, - ColumnString& to_column) const; + // Skip rows in source_null_map and insert a default value for each skipped row. + virtual Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const; virtual Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const { diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 2b60cc5f083186..4bc8c51edcea92 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -766,9 +766,9 @@ Status DataTypeStringSerDeBase::write_column_to_orc( } template -Status DataTypeStringSerDeBase::serialize_column_to_jsonb(const IColumn& from_column, - int64_t row_num, - JsonbWriter& writer) const { +Status DataTypeStringSerDeBase::serialize_column_to_jsonb( + const IColumn& from_column, int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { if constexpr (!std::is_same_v) { return Status::NotSupported( "DataTypeStringSerDeBase only supports ColumnString for serialize_column_to_jsonb"); diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index f90b27b1555452..46ca26a6676388 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -178,7 +178,8 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_pb(IColumn& column, const PValues& arg) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index 04f378104f5d9c..23b6bf3103cb36 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -385,7 +385,8 @@ Status DataTypeStructSerDe::serialize_one_cell_to_hive_text( } Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const { + JsonbWriter& writer, + const FormatOptions& options) const { const auto& struct_column = assert_cast(from_column); if (!writer.writeStartObject()) { @@ -403,7 +404,7 @@ Status DataTypeStructSerDe::serialize_column_to_jsonb(const IColumn& from_column } // write value RETURN_IF_ERROR(elem_serdes_ptrs[i]->serialize_column_to_jsonb(struct_column.get_column(i), - row_num, writer)); + row_num, writer, options)); } if (!writer.writeEndObject()) { diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index a6a810940bc756..7cb7eac4d57f72 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -114,7 +114,8 @@ class DataTypeStructSerDe : public DataTypeSerDe { const FormatOptions& options) const override; Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, - JsonbWriter& writer) const override; + JsonbWriter& writer, + const FormatOptions& options) const override; Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, CastParameters& castParms) const override; diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp index 4e9ae6f872d1cd..20c7ba92936d22 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp @@ -34,6 +34,7 @@ #include "core/arena.h" #include "core/assert_cast.h" #include "core/column/column_const.h" +#include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/column/variant_v2/column_variant_v2_typed_column.h" @@ -48,6 +49,7 @@ #include "core/value/variant/variant_batch_builder.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" +#include "util/jsonb_document.h" #include "util/jsonb_writer.h" #include "util/mysql_row_buffer.h" @@ -479,6 +481,47 @@ void DataTypeVariantV2SerDe::read_one_cell_from_jsonb(IColumn& column, destination(column).insert_encoded_batch(block); } +// to_json and CAST( AS JSON) write Variant values here, with the same session time zone +// as CAST( AS JSON). +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb(const IColumn& from_column, + int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const { + RETURN_IF_CATCH_EXCEPTION({ + const size_t row = checked_row(row_num); + JsonbWriter document; + visit_variant_v2_values( + from_column, row, row + 1, {}, [](size_t) {}, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, document, {.timezone = options.timezone}); + }); + const JsonbDocument* doc = nullptr; + THROW_IF_ERROR(JsonbDocument::checkAndCreateDocument( + document.getOutput()->getBuffer(), document.getOutput()->getSize(), &doc)); + require_jsonb_write(writer.writeValue(doc->getValue()), "Variant value"); + }); + return Status::OK(); +} + +Status DataTypeVariantV2SerDe::serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map) const { + RETURN_IF_CATCH_EXCEPTION({ + JsonbWriter writer; + visit_variant_v2_values( + from_column, 0, from_column.size(), + source_null_map + ? std::span(source_null_map, from_column.size()) + : std::span(), + [&](size_t) { to_column.insert_default(); }, + [&](size_t, VariantRef value) { + variant_to_jsonb(value, writer, {.timezone = options.timezone}); + to_column.insert_data(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()); + }); + }); + return Status::OK(); +} + namespace { DorisVector json_lengths(const IColumn& column, size_t start, size_t end, diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.h b/be/src/core/data_type_serde/data_type_variant_v2_serde.h index ff2565784b6e6e..c4bc849ca8db6a 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.h +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.h @@ -53,6 +53,13 @@ class DataTypeVariantV2SerDe final : public DataTypeSerDe { const FormatOptions& options) const override; void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; + Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num, + JsonbWriter& writer, + const FormatOptions& options) const override; + Status serialize_column_to_jsonb_vector( + const IColumn& from_column, ColumnString& to_column, const FormatOptions& options, + const NullMap::value_type* source_null_map = nullptr) const override; + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; diff --git a/be/src/core/value/variant/variant_canonical.cpp b/be/src/core/value/variant/variant_canonical.cpp index 07b8dcc5965fc6..ffa0dcae0c8cc8 100644 --- a/be/src/core/value/variant/variant_canonical.cpp +++ b/be/src/core/value/variant/variant_canonical.cpp @@ -30,9 +30,12 @@ #include #include +#include "common/compare.h" #include "common/exception.h" +#include "core/uint128.h" #include "core/value/variant/variant_field.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/common/int_exp.h" #include "exec/common/sip_hash.h" #include "util/hash_util.hpp" #include "util/utf8_check.h" @@ -256,77 +259,253 @@ ObjectEntry object_entry_at(VariantRef object, uint32_t index, StringRef previou return {.key = key, .value = child}; } -bool scalar_equals(const NormalizedValue& left, const NormalizedValue& right) { +bool is_numeric(CanonicalKind kind) { + return kind == CanonicalKind::EXACT_INTEGER || kind == CanonicalKind::DECIMAL || + kind == CanonicalKind::FLOATING; +} + +struct ExactNumber { + unsigned __int128 magnitude; + uint8_t scale; + bool negative; +}; + +ExactNumber exact_number(const NormalizedValue& value) { + DCHECK(value.kind == CanonicalKind::EXACT_INTEGER || value.kind == CanonicalKind::DECIMAL); + return {.magnitude = variant_unsigned_magnitude(value.integer), + .scale = static_cast(value.kind == CanonicalKind::DECIMAL ? value.scale : 0), + .negative = value.integer < 0}; +} + +// A Variant decimal scale is validated to [0, 38] before the value is stored +// (variant_scalar.cpp, variant_value.cpp), so the shared exp10 table, which covers 0 to 76, always +// holds these exponents. +UInt256 power_of_five(uint8_t exponent) { + // 10^k = 2^k * 5^k, so 5^k is the table entry shifted right by k, exactly. + return static_cast(common::exp10_i256(exponent)) >> exponent; +} + +UInt256 power_of_ten(uint8_t exponent) { + return static_cast(common::exp10_i256(exponent)); +} + +uint16_t bit_width(const UInt256& value) { + constexpr unsigned LIMB_BITS = sizeof(value.items[0]) * 8; + // big(index) walks the limbs most significant first, so the first non-zero one carries the + // width; scanning limbs costs 4 iterations instead of up to 256 single-bit shifts. + for (unsigned index = 0; index < UInt256::item_count; ++index) { + const auto limb = value.items[UInt256::_impl::big(index)]; + if (limb != 0) { + return static_cast((UInt256::item_count - index) * LIMB_BITS - + static_cast(std::countl_zero(limb))); + } + } + return 0; +} + +int compare_exact_magnitudes(const ExactNumber& left, const ExactNumber& right) { + // Equal scales are the common case - two JSON integers both have scale 0 - and need no + // rescaling, so they never reach the 256-bit path below. + if (left.scale == right.scale) { + return Compare::compare(left.magnitude, right.magnitude); + } + UInt256 scaled_left {left.magnitude}; + UInt256 scaled_right {right.magnitude}; + if (left.scale < right.scale) { + scaled_left *= power_of_ten(right.scale - left.scale); + } else { + scaled_right *= power_of_ten(left.scale - right.scale); + } + return Compare::compare(scaled_left, scaled_right); +} + +struct FloatingNumber { + uint64_t significand; + int16_t binary_exponent; + bool negative; + bool infinity; + bool nan; +}; + +FloatingNumber floating_number(uint64_t bits) { + constexpr uint64_t FRACTION_MASK = (uint64_t {1} << 52) - 1; + constexpr uint16_t EXPONENT_MASK = (uint16_t {1} << 11) - 1; + const auto exponent = static_cast((bits >> 52) & EXPONENT_MASK); + const uint64_t fraction = bits & FRACTION_MASK; + if (exponent == EXPONENT_MASK) { + return {.significand = 0, + .binary_exponent = 0, + .negative = (bits >> 63) != 0, + .infinity = fraction == 0, + .nan = fraction != 0}; + } + if (exponent == 0) { + return {.significand = fraction, + .binary_exponent = -1074, + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; + } + return {.significand = (uint64_t {1} << 52) | fraction, + .binary_exponent = static_cast(static_cast(exponent) - 1023 - 52), + .negative = (bits >> 63) != 0, + .infinity = false, + .nan = false}; +} + +int compare_floating_exact_magnitudes(const FloatingNumber& floating, const ExactNumber& exact) { + DCHECK(!floating.nan && !floating.infinity); + DCHECK_NE(floating.significand, 0); + DCHECK(exact.magnitude != 0); + UInt256 floating_scaled = UInt256(floating.significand) * power_of_five(exact.scale); + UInt256 exact_scaled {exact.magnitude}; + const int16_t binary_shift = floating.binary_exponent + exact.scale; + if (binary_shift >= 0) { + const uint16_t floating_width = bit_width(floating_scaled); + if (floating_width + binary_shift > bit_width(exact_scaled)) { + return 1; + } + floating_scaled <<= binary_shift; + } else { + const uint16_t exact_width = bit_width(exact_scaled); + const auto shift = static_cast(-binary_shift); + if (exact_width + shift > bit_width(floating_scaled)) { + return -1; + } + exact_scaled <<= shift; + } + return Compare::compare(floating_scaled, exact_scaled); +} + +int compare_numeric(const NormalizedValue& left, const NormalizedValue& right) { + if (left.kind != CanonicalKind::FLOATING && right.kind != CanonicalKind::FLOATING) { + const ExactNumber exact_left = exact_number(left); + const ExactNumber exact_right = exact_number(right); + if (exact_left.negative != exact_right.negative) { + return exact_left.negative ? -1 : 1; + } + int result = compare_exact_magnitudes(exact_left, exact_right); + return exact_left.negative ? -result : result; + } + if (left.kind == CanonicalKind::FLOATING && right.kind == CanonicalKind::FLOATING) { + const FloatingNumber floating_left = floating_number(left.floating_bits); + const FloatingNumber floating_right = floating_number(right.floating_bits); + if (floating_left.nan || floating_right.nan) { + return Compare::compare(floating_left.nan, floating_right.nan); + } + const auto value_left = std::bit_cast(left.floating_bits); + const auto value_right = std::bit_cast(right.floating_bits); + return Compare::compare(value_left, value_right); + } + + const bool floating_on_left = left.kind == CanonicalKind::FLOATING; + const FloatingNumber floating = + floating_number((floating_on_left ? left : right).floating_bits); + const ExactNumber exact = exact_number(floating_on_left ? right : left); + int result; + if (floating.nan) { + result = 1; + } else if (floating.infinity) { + result = floating.negative ? -1 : 1; + } else if (exact.magnitude == 0) { + result = floating.negative ? -1 : 1; + } else if (floating.negative != exact.negative) { + result = floating.negative ? -1 : 1; + } else { + result = compare_floating_exact_magnitudes(floating, exact); + if (floating.negative) { + result = -result; + } + } + return floating_on_left ? result : -result; +} + +int scalar_compare(const NormalizedValue& left, const NormalizedValue& right) { + if (is_numeric(left.kind) && is_numeric(right.kind)) { + const int numeric_result = compare_numeric(left, right); + if (numeric_result != 0) { + return numeric_result; + } + return Compare::compare(left.kind, right.kind); + } + if (left.kind != right.kind) { + return Compare::compare(left.kind, right.kind); + } switch (left.kind) { case CanonicalKind::NULL_VALUE: - return true; + return 0; case CanonicalKind::BOOL: - return left.boolean == right.boolean; + return Compare::compare(left.boolean, right.boolean); case CanonicalKind::EXACT_INTEGER: case CanonicalKind::DATE: case CanonicalKind::TIMESTAMP_TZ: case CanonicalKind::TIMESTAMP_NTZ: case CanonicalKind::TIME: - return left.integer == right.integer; + return Compare::compare(left.integer, right.integer); case CanonicalKind::DECIMAL: - return left.integer == right.integer && left.scale == right.scale; + DCHECK(false) << "Decimal should use numeric comparison"; + return 0; case CanonicalKind::FLOATING: - return left.floating_bits == right.floating_bits; + DCHECK(false) << "Floating point should use numeric comparison"; + return 0; case CanonicalKind::STRING: case CanonicalKind::BINARY: - return left.bytes == right.bytes; + return Compare::compare(left.bytes, right.bytes); case CanonicalKind::UUID: - return left.uuid == right.uuid; + return Compare::compare(left.uuid, right.uuid); case CanonicalKind::OBJECT: case CanonicalKind::ARRAY: break; } - DCHECK(false) << "Container reached scalar equality"; - return false; + DCHECK(false) << "Container reached scalar comparison"; + return 0; } -bool equals_node(VariantRef left, VariantRef right, uint32_t depth) { +int compare_node(VariantRef left, VariantRef right, uint32_t depth) { require_depth(depth); require_exact_value(left); require_exact_value(right); const NormalizedValue normalized_left = normalize_value(left); const NormalizedValue normalized_right = normalize_value(right); if (normalized_left.kind != normalized_right.kind) { - return false; + if (!is_numeric(normalized_left.kind) || !is_numeric(normalized_right.kind)) { + return Compare::compare(normalized_left.kind, normalized_right.kind); + } } if (normalized_left.kind == CanonicalKind::ARRAY) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } - for (uint32_t index = 0; index < count; ++index) { - if (!equals_node(left.array_at(index), right.array_at(index), depth + 1)) { - return false; + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { + const int result = compare_node(left.array_at(index), right.array_at(index), depth + 1); + if (result != 0) { + return result; } } - return true; + return Compare::compare(left_count, right_count); } if (normalized_left.kind == CanonicalKind::OBJECT) { - const uint32_t count = left.num_elements(); - if (count != right.num_elements()) { - return false; - } + const uint32_t left_count = left.num_elements(); + const uint32_t right_count = right.num_elements(); StringRef previous_left; StringRef previous_right; - for (uint32_t index = 0; index < count; ++index) { + for (uint32_t index = 0; index < std::min(left_count, right_count); ++index) { const ObjectEntry left_entry = object_entry_at(left, index, previous_left, index != 0); const ObjectEntry right_entry = object_entry_at(right, index, previous_right, index != 0); - if (left_entry.key != right_entry.key || - !equals_node(left_entry.value, right_entry.value, depth + 1)) { - return false; + const int key_result = Compare::compare(left_entry.key, right_entry.key); + if (key_result != 0) { + return key_result; + } + const int value_result = compare_node(left_entry.value, right_entry.value, depth + 1); + if (value_result != 0) { + return value_result; } previous_left = left_entry.key; previous_right = right_entry.key; } - return true; + return Compare::compare(left_count, right_count); } - return scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); } template @@ -355,7 +534,7 @@ void update_bytes(Sink& sink, StringRef bytes) { } template -void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { +inline void hash_normalized_scalar(const NormalizedValue& normalized, Sink& sink) { switch (normalized.kind) { case CanonicalKind::NULL_VALUE: return; @@ -1004,7 +1183,11 @@ void VariantCrc32cHashSink::update(const char* data, size_t size) { } bool canonical_equals(VariantRef left, VariantRef right) { - return equals_node(left, right, 0); + return canonical_compare(left, right) == 0; +} + +int canonical_compare(VariantRef left, VariantRef right) { + return compare_node(left, right, 0); } template @@ -1018,10 +1201,21 @@ template void canonical_hash(VariantRef value, VariantCrc3 template void canonical_hash(VariantRef value, VariantCrc32cHashSink& sink); bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right) { + return canonical_compare(left, right) == 0; +} + +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right) { const NormalizedValue normalized_left = VariantScalarAdapter::normalize(left); const NormalizedValue normalized_right = VariantScalarAdapter::normalize(right); - return normalized_left.kind == normalized_right.kind && - scalar_equals(normalized_left, normalized_right); + return scalar_compare(normalized_left, normalized_right); +} + +int canonical_compare(const VariantScalarRef& left, VariantRef right) { + require_exact_value(right); + // A scalar never has a container kind, so an object or array on the right is decided by the + // kind order inside scalar_compare(), as compare_node() decides it, without walking the + // container. + return scalar_compare(VariantScalarAdapter::normalize(left), normalize_value(right)); } template diff --git a/be/src/core/value/variant/variant_canonical.h b/be/src/core/value/variant/variant_canonical.h index 286b43f94ec723..15d78e1334613d 100644 --- a/be/src/core/value/variant/variant_canonical.h +++ b/be/src/core/value/variant/variant_canonical.h @@ -35,6 +35,11 @@ struct VariantScalarAdapter; bool canonical_equals(const VariantScalarRef& left, const VariantScalarRef& right); +// Returns a negative value, zero, or a positive value when left is respectively less than, equal +// to, or greater than right in the canonical Variant total order. Zero is exactly equivalent to +// canonical_equals(). +int canonical_compare(const VariantScalarRef& left, const VariantScalarRef& right); + template void canonical_hash(const VariantScalarRef& value, Sink& sink); @@ -77,8 +82,28 @@ class VariantCrc32cHashSink { // Compares logical Variant values. Numeric normalization is deliberately limited to the exact // integer domain that canonical Variant decimal16 can encode: [-(10^38-1), +(10^38-1)]. A finite // integral float/double outside that domain remains floating so canonical arena bytes stay valid. +// +// The canonical equality contract for numbers: +// - an integer, a decimal with a zero fraction and an integral float/double in the domain above +// are one value: 1, 1.0 (decimal) and 1.0 (double) are equal; +// - a decimal and a float/double with a fraction are never equal, even when they print the same: +// DECIMAL 1.5 != DOUBLE 1.5. They order by numeric value, and at a numeric tie the decimal +// sorts first. Equating them would need a lossless common form, and the exact value of most +// doubles (0.1 for one) does not fit the 38 digits of a Variant decimal. +// canonical_equals(), canonical_compare() == 0, canonical_hash() and the canonical serialized key +// (GROUP BY, DISTINCT, join build) all implement this one rule, and sort peers follow it. A change +// to the rule has to change all of them together. bool canonical_equals(VariantRef left, VariantRef right); +// Container values are compared lexicographically after scalar normalization. Object keys use +// their canonical byte order. Numeric kinds share a numeric order, with canonical kind as a +// deterministic tie-breaker when two numerically equal values are not canonically equal. +int canonical_compare(VariantRef left, VariantRef right); + +// Compares a typed scalar with an encoded root in the same order, without encoding the scalar: +// canonical_compare(left, right) == canonical_compare(encode(left), right). +int canonical_compare(const VariantScalarRef& left, VariantRef right); + // Hashes canonical logical tokens without re-encoding the value. Supported production sinks are // SipHash, VariantXxHashSink, VariantCrc32HashSink, and VariantCrc32cHashSink; all four are explicit // instantiations of the same traversal skeleton. diff --git a/be/src/exec/sort/sort_block.cpp b/be/src/exec/sort/sort_block.cpp index 69277b0c46aa80..c45b5d95aa53e0 100644 --- a/be/src/exec/sort/sort_block.cpp +++ b/be/src/exec/sort/sort_block.cpp @@ -21,9 +21,17 @@ #include "exec/sort/sort_block.h" #include "core/block/column_with_type_and_name.h" +#include "core/column/variant_v2/column_variant_v2.h" namespace doris { #include "common/compile_check_begin.h" + +void ColumnSorter::sort_column(const ColumnVariantV2& column, EqualFlags& flags, + IColumn::Permutation& perms, EqualRange& range, + bool last_column) const { + _sort_by_default(column, flags, perms, range, last_column); +} + ColumnsWithSortDescriptions get_columns_with_sort_description(const Block& block, const SortDescription& description) { size_t size = description.size(); diff --git a/be/src/exec/sort/sort_block.h b/be/src/exec/sort/sort_block.h index d7da474e246add..19bf3c9a1d0d66 100644 --- a/be/src/exec/sort/sort_block.h +++ b/be/src/exec/sort/sort_block.h @@ -51,6 +51,7 @@ template class ColumnDecimal; template class ColumnVector; +class ColumnVariantV2; } // namespace doris namespace doris { @@ -253,6 +254,8 @@ class ColumnSorter { EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); } + void sort_column(const ColumnVariantV2& column, EqualFlags& flags, IColumn::Permutation& perms, + EqualRange& range, bool last_column) const; void sort_column(const ColumnVarbinary& column, EqualFlags& flags, IColumn::Permutation& perms, EqualRange& range, bool last_column) const { _sort_by_default(column, flags, perms, range, last_column); @@ -388,7 +391,8 @@ class ColumnSorter { !std::is_same_v && !std::is_same_v && !std::is_same_v && - !std::is_same_v) { + !std::is_same_v && + !std::is_same_v) { auto value_a = column.get_data()[a]; auto value_b = column.get_data()[b]; return Compare::compare(value_a, value_b); diff --git a/be/src/exprs/function/cast/cast_to_jsonb.h b/be/src/exprs/function/cast/cast_to_jsonb.h index ca90ff7d877567..ac834557d45261 100644 --- a/be/src/exprs/function/cast/cast_to_jsonb.h +++ b/be/src/exprs/function/cast/cast_to_jsonb.h @@ -24,6 +24,7 @@ #include "core/value/jsonb_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_string.h" +#include "runtime/runtime_state.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -133,7 +134,7 @@ WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type, } return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t input_rows_count, const NullMap::value_type*) { + uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { CastParameters params; params.is_strict = context->enable_strict_mode(); @@ -231,13 +232,15 @@ WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const Dat } return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t input_rows_count, const NullMap::value_type*) { + uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) { // same as to_json function auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; - RETURN_IF_ERROR( - from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column)); + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); + RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, + options, null_map)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); }; diff --git a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp index 7eea84bbdccd6b..f5e6ede566b1b7 100644 --- a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp +++ b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp @@ -425,7 +425,12 @@ Status cast_scalar_to_variant(const ColumnPtr& source, const DataTypePtr& source } ColumnPtr null_map = std::move(nulls); ColumnPtr nullable = ColumnNullable::create(source, null_map); - *output = ColumnVariantV2::create_typed(std::move(nullable), source_type); + auto typed = ColumnVariantV2::create_typed(std::move(nullable), source_type); + // A scalar column can hold values Variant cannot, such as a string that is not UTF-8. Reject + // them here, as encoding would, so the typed state never compares or hashes a value that its + // encoded form could not hold. The CAST wrapper turns the exception into the statement error. + typed->validate_typed_rows(); + *output = std::move(typed); return Status::OK(); } diff --git a/be/src/exprs/function/function_to_json.cpp b/be/src/exprs/function/function_to_json.cpp index bae1f6767af174..d8956c0d3d2d75 100644 --- a/be/src/exprs/function/function_to_json.cpp +++ b/be/src/exprs/function/function_to_json.cpp @@ -16,7 +16,10 @@ // under the License. #include "core/data_type/data_type_jsonb.h" +#include "core/data_type_serde/data_type_serde.h" #include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "runtime/runtime_state.h" namespace doris { #include "common/compile_check_begin.h" @@ -43,8 +46,10 @@ class FunctionToJson : public IFunction { auto to_column = ColumnString::create(); auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde(); auto from_column = block.get_by_position(arguments[0]).column; - RETURN_IF_ERROR( - from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column)); + DataTypeSerDe::FormatOptions options; + options.timezone = &context->state()->timezone_obj(); + RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column, + options)); block.get_by_position(result).column = std::move(to_column); return Status::OK(); } diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index efb68f5f6325c2..e054aba7dd812b 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -872,12 +872,12 @@ void append_value(VariantRef value, const DataTypePtr& target_type, IColumn* tar Status stringify_complex_column(const DataTypePtr& source_type, const ColumnNullable& source, ColumnPtr* result) { + DataTypeSerDe::FormatOptions options; auto jsonb = ColumnString::create(); RETURN_IF_ERROR(source_type->get_serde()->serialize_column_to_jsonb_vector( - source.get_nested_column(), *jsonb)); + source.get_nested_column(), *jsonb, options)); auto strings = ColumnString::create(); - DataTypeSerDe::FormatOptions options; DataTypeJsonb().get_serde()->to_string_batch(*jsonb, *strings, options); *result = ColumnNullable::create(std::move(strings), source.get_null_map_column().clone_resized(source.size())); diff --git a/be/test/core/column/column_variant_v2_test.cpp b/be/test/core/column/column_variant_v2_test.cpp index 9f5ff6ce1d27de..2f21e957c749a4 100644 --- a/be/test/core/column/column_variant_v2_test.cpp +++ b/be/test/core/column/column_variant_v2_test.cpp @@ -63,12 +63,14 @@ #include "exec/common/hash_table/hash_map_context.h" #include "exec/common/hash_table/string_hash_map.h" #include "exec/common/sip_hash.h" +#include "exec/sort/sort_block.h" #include "exprs/function/parse/variant_jsonb_parse.h" #include "exprs/function/parse/variant_string_parse.h" #include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" #include "util/jsonb_writer.h" #include "util/variant/variant_test_utils.h" +#include "vec/core/hybrid_sorter.h" namespace doris { namespace { @@ -284,6 +286,50 @@ class CountingShreddedState final : public VariantShreddedState { ColumnVariantV2::MutablePtr _serialized; }; +// A complete shredded state over known rows that counts how often its canonical rows are asked for. +class MaterializingShreddedState final : public VariantShreddedState { +public: + MaterializingShreddedState(ColumnPtr rows, std::shared_ptr materialized_calls) + : _rows(std::move(rows)), _materialized_calls(std::move(materialized_calls)) {} + + size_t size() const override { return _rows->size(); } + size_t byte_size() const override { return 0; } + size_t allocated_bytes() const override { return 0; } + void sanity_check() const override {} + void for_each_subcolumn(const IColumn::ImutableColumnCallback&) const override {} + std::shared_ptr filter(const IColumn::Filter&, ssize_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_range(size_t, size_t) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + std::shared_ptr select_indices(const uint32_t*, + const uint32_t*) const override { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "row selection is not under test"); + } + bool can_materialize() const override { return true; } + bool try_append(const VariantShreddedState&) override { return false; } + std::optional find_typed_value( + std::span) const override { + return std::nullopt; + } + std::optional find_normalized_value( + std::span) const override { + return std::nullopt; + } + const ColumnVariantV2& materialized_column() const override { + ++*_materialized_calls; + return assert_cast(*_rows); + } + const ColumnVariantV2& serialized_column() const override { + return assert_cast(*_rows); + } + +private: + ColumnPtr _rows; + std::shared_ptr _materialized_calls; +}; + class ValidatingShreddedState final : public VariantShreddedState { public: enum class TypedLookup { MISS, ERROR }; @@ -587,6 +633,20 @@ void validate_encoded_column(const ColumnVariantV2& column) { } } +// The encoded twin of a typed column. insert_range_from() into an empty column keeps the source's +// typed state, so a test that wants the other representation has to encode explicitly; both +// physical states are asserted so a variable name cannot stand in for coverage. +ColumnVariantV2::MutablePtr encoded_copy_of(const ColumnVariantV2& typed) { + EXPECT_TRUE(typed.is_typed()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(typed, 0, typed.size()); + encoded->ensure_encoded(); + EXPECT_FALSE(encoded->is_typed()); + EXPECT_FALSE(encoded->is_shredded()); + EXPECT_EQ(encoded->size(), typed.size()); + return encoded; +} + void ensure_typed_fields_match_direct_encoding(ColumnVariantV2& column) { ASSERT_TRUE(column.is_typed()); std::vector fields(column.size()); @@ -1751,6 +1811,47 @@ TEST(ColumnVariantV2Test, CompositeShreddedSizeDoesNotRecountSegments) { EXPECT_EQ(*size_calls, 0); } +TEST(ColumnVariantV2Test, EraseRemovesRowsInEveryState) { + // Encoded rows with two metadata dictionaries: the ids of the remaining rows stay valid. + auto encoded = ColumnVariantV2::create(); + for (const char* json : {R"({"a":1})", "2", R"({"b":3})", "4", R"("five")"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + encoded->erase(0, 2); + ASSERT_EQ(encoded->size(), 3); + EXPECT_EQ(json_at(*encoded, 0), R"({"b":3})"); + EXPECT_EQ(json_at(*encoded, 2), R"("five")"); + encoded->erase(1, 1); + ASSERT_EQ(encoded->size(), 2); + EXPECT_EQ(json_at(*encoded, 1), R"("five")"); + encoded->erase(2, 0); + EXPECT_EQ(encoded->size(), 2); + validate_encoded_column(*encoded); + + constexpr std::array VALUES {7, 8, 9, 10}; + constexpr std::array NULLS {0, 1, 0, 0}; + auto typed = typed_int32(VALUES, NULLS); + typed->erase(0, 2); + EXPECT_TRUE(typed->is_typed()); + constexpr std::array REMAINING {9, 10}; + constexpr std::array REMAINING_NULLS {0, 0}; + expect_int32_rows(*typed, REMAINING, REMAINING_NULLS); + + auto rows = ColumnVariantV2::create(); + for (const char* json : {"1", "2", "3"}) { + insert_encoded_field(*rows, encode_json(json)); + } + auto calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(rows->get_ptr(), calls)); + shredded->erase(0, 1); + EXPECT_FALSE(shredded->is_shredded()); + ASSERT_EQ(shredded->size(), 2); + EXPECT_EQ(json_at(*shredded, 0), "2"); + EXPECT_EQ(json_at(*shredded, 1), "3"); + EXPECT_EQ(rows->size(), 3); +} + TEST(ColumnVariantV2Test, CompositeValidatesSegmentsAfterFirstTypedMiss) { auto first_calls = std::make_shared(0); auto second_calls = std::make_shared(0); @@ -2676,6 +2777,30 @@ TEST(ColumnVariantV2Test, TypedEnsureEncodedAllScalarMappings) { ASSERT_EQ(zero_dates.size(), 2); EXPECT_EQ(zero_dates.get_data()[0], valid_date); EXPECT_EQ(zero_dates.get_data()[1], invalid_date); + + // validate_typed_rows() is the check CAST runs on arbitrary scalars. It reads the rows the + // typed null map leaves visible and nothing else. + EXPECT_THROW(zero_date->validate_typed_rows(), Exception); + auto masked_nested = ColumnDateV2::create(); + masked_nested->insert_value(valid_date); + masked_nested->insert_value(invalid_date); + auto masked_nulls = ColumnUInt8::create(); + masked_nulls->insert_value(0); + masked_nulls->insert_value(1); + auto masked = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(masked_nested), std::move(masked_nulls)), + std::make_shared()); + EXPECT_NO_THROW(masked->validate_typed_rows()); + masked->ensure_encoded(); + validate_encoded_column(*masked); + EXPECT_TRUE(masked->get_value_ref(1).is_null()); + + constexpr std::array INVALID_UTF8_ROWS {"ok", "\xFF"}; + constexpr std::array INVALID_UTF8_VISIBLE {0, 0}; + constexpr std::array INVALID_UTF8_HIDDEN {0, 1}; + EXPECT_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_VISIBLE)->validate_typed_rows(), + Exception); + EXPECT_NO_THROW(typed_strings(INVALID_UTF8_ROWS, INVALID_UTF8_HIDDEN)->validate_typed_rows()); } TEST(ColumnVariantV2Test, TypedRowTransformsCloneAndCow) { @@ -2813,6 +2938,8 @@ TEST(ColumnVariantV2Test, TypedMutatorsDetachSharedNestedColumn) { constexpr std::array FIRST_TWO_NULLS {}; exercise([](ColumnVariantV2& column) { column.pop_back(1); }, FIRST_TWO, FIRST_TWO_NULLS); exercise([](ColumnVariantV2& column) { column.resize(2); }, FIRST_TWO, FIRST_TWO_NULLS); + constexpr std::array LAST_TWO {2, 3}; + exercise([](ColumnVariantV2& column) { column.erase(0, 1); }, LAST_TWO, FIRST_TWO_NULLS); constexpr std::array FILTERED {1, 3}; constexpr std::array FILTERED_NULLS {}; exercise( @@ -3035,6 +3162,45 @@ TEST(ColumnVariantV2Test, MixedEncodedTypedInsertAndGatherPreserveCanonicalRows) expect_canonical_rows_equal(*typed, *encoded); } +TEST(ColumnVariantV2Test, NullableBatchSerializationMatchesRowSerialization) { + constexpr std::array VALUES {42, 0, -7}; + constexpr std::array NULLS {0, 1, 0}; + auto typed = typed_int32(VALUES, NULLS); + auto encoded = ColumnVariantV2::create(); + insert_encoded_field(*encoded, encoded_integer(42, 4)); + insert_encoded_field(*encoded, encode_json("null")); + insert_encoded_field(*encoded, encoded_integer(-7, 4)); + for (const auto* column : {typed.get(), encoded.get()}) { + for (const bool has_null : {false, true}) { + std::array buffers; + std::array keys; + for (size_t row = 0; row < 3; ++row) { + buffers[row].assign(8 + column->get_max_row_byte_size(), '\0'); + buffers[row][0] = 'x'; + keys[row] = {buffers[row].data(), 1}; + } + constexpr std::array NO_NULLS {0, 0, 0}; + auto nullable = + ColumnNullable::create(column->clone_resized(3), ColumnUInt8::create(3, 0)); + for (size_t row = 0; row < 3; ++row) { + nullable->get_null_map_data()[row] = has_null ? NULLS[row] : NO_NULLS[row]; + } + nullable->serialize_vec(keys.data(), 3); + for (size_t row = 0; row < 3; ++row) { + std::string expected = "x"; + const bool is_null = has_null && NULLS[row]; + expected.push_back(is_null ? 1 : 0); + if (!is_null) { + std::string cell(column->serialize_size_at(row), '\0'); + column->serialize_impl(cell.data(), row); + expected += cell; + } + EXPECT_EQ(std::string(keys[row].data, keys[row].size), expected); + } + } + } +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive E/T adapter matrix. TEST(ColumnVariantV2Test, TypedCanonicalHashCrcAndArenaMatchEncoded) { constexpr std::array VALUES {42, 0, -7}; @@ -3393,21 +3559,309 @@ TEST(ColumnVariantV2Test, ETCrossCheckTemporalClassMatrix) { ntz_representations[2], 0); } -TEST(ColumnVariantV2Test, TypedPhysicalAndOrderingInterfacesStayUnsupported) { - constexpr std::array VALUES {1}; - constexpr std::array NULLS {0}; +TEST(ColumnVariantV2Test, TypedComparisonBoundariesMatchEncodedWithoutMaterializing) { + auto check = [](const ColumnVariantV2::MutablePtr& typed) { + ASSERT_TRUE(typed->is_typed()); + const auto* original = &typed->typed_column(); + auto encoded = encoded_copy_of(*typed); + ASSERT_FALSE(encoded->is_typed()); + for (size_t right = 0; right < typed->size(); ++right) { + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + const int expected = encoded->compare_at(left, right, *encoded, hint); + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), expected); + EXPECT_EQ(typed->compare_at(left, right, *encoded, hint), expected); + EXPECT_EQ(encoded->compare_at(left, right, *typed, hint), expected); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(right, *typed, hint, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(right, *encoded, hint, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + } + // Prove the tested input did not silently become encoded during comparison. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + check(ColumnVariantV2::create_typed( + nullable_fixed({0, -1, 9007199254740993LL, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-std::numeric_limits::infinity(), -0.0, 0.0, 1.5, + std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + std::bit_cast(uint64_t {0xfff8000000000001ULL}), 0.0}, + {0, 0, 0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 2, {Decimal64 {-150}, Decimal64 {0}, Decimal64 {150}, Decimal64 {0}}, + {0, 0, 0, 1}), + std::make_shared(18, 2))); + const auto date = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 2, 29)); + const auto next = + DateV2Value::create_from_olap_date(pack_olap_date(2024, 3, 1)); + check(ColumnVariantV2::create_typed(nullable_fixed>( + {date, next, date}, {0, 0, 1}), + std::make_shared())); + // Address order differs from textual Variant order for these values. + IPv4 ip4_a {}, ip4_b {}; + ASSERT_TRUE(IPv4Value::from_string(ip4_a, "10.0.0.2")); + ASSERT_TRUE(IPv4Value::from_string(ip4_b, "10.0.0.10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip4_a, ip4_b, ip4_a}, {0, 0, 1}), + std::make_shared())); + IPv6 ip6_a {}, ip6_b {}; + ASSERT_TRUE(IPv6Value::from_string(ip6_a, "2001:db8::2")); + ASSERT_TRUE(IPv6Value::from_string(ip6_b, "2001:db8::10")); + check(ColumnVariantV2::create_typed( + nullable_fixed({ip6_a, ip6_b, ip6_a}, {0, 0, 1}), + std::make_shared())); + // LARGEINT magnitudes above 10^38 - 1 are represented as Variant strings. + const __int128 limit = power_of_ten_i128(38); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-limit, Int128 {0}, limit - 1, limit, -(limit - 1), Int128 {0}}, + {0, 0, 0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_fixed({1, 0, 1}, {0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed({-1, 0, 127, 0}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_fixed( + {-1.25F, 0.5F, std::numeric_limits::quiet_NaN(), 0.0F}, {0, 0, 0, 1}), + std::make_shared())); + check(ColumnVariantV2::create_typed( + nullable_decimal( + 4, + {Decimal128V3 {Int128 {-15000}}, Decimal128V3 {Int128 {0}}, + Decimal128V3 {Int128 {15000}}, Decimal128V3 {Int128 {0}}}, + {0, 0, 0, 1}), + std::make_shared(38, 4))); + const auto datetime_a = + DateV2Value::create_from_olap_datetime(19700101000001ULL); + const auto datetime_b = + DateV2Value::create_from_olap_datetime(20240229123456ULL); + check(ColumnVariantV2::create_typed( + nullable_fixed>( + {datetime_b, datetime_a, datetime_b}, {0, 0, 1}), + std::make_shared(6))); + // Bytes at or above 0x80 pin the unsigned string order. + constexpr std::array STRINGS {"a", "\xC3\xA9", "z", ""}; + constexpr std::array STRING_NULLS {0, 0, 0, 1}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, TypedComparisonAcrossTypeParametersMatchesEncoded) { + // DECIMAL(18,2) and DECIMAL(18,3) share a column type but not a scale. + auto left = ColumnVariantV2::create_typed(nullable_decimal( + 2, {Decimal64 {150}, Decimal64 {-5}}, {0, 0}), + std::make_shared(18, 2)); + auto right = + ColumnVariantV2::create_typed(nullable_decimal( + 3, {Decimal64 {1400}, Decimal64 {-50}}, {0, 0}), + std::make_shared(18, 3)); + auto left_encoded = encoded_copy_of(*left); + auto right_encoded = encoded_copy_of(*right); + for (size_t right_row = 0; right_row < right->size(); ++right_row) { + for (size_t left_row = 0; left_row < left->size(); ++left_row) { + EXPECT_EQ(left->compare_at(left_row, right_row, *right, 1), + left_encoded->compare_at(left_row, right_row, *right_encoded, 1)); + } + for (int direction : {-1, 1}) { + std::vector typed_cmp(left->size(), 0); + std::vector encoded_cmp(left->size(), 0); + IColumn::Filter typed_filter(left->size(), 0); + IColumn::Filter encoded_filter(left->size(), 0); + left->compare_internal(right_row, *right, 1, direction, typed_cmp, typed_filter.data()); + left_encoded->compare_internal(right_row, *right_encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } +} + +TEST(ColumnVariantV2Test, TypedAgainstEncodedRootsMatchesEncoded) { + // The encoded side holds every kind of root a typed scalar can meet, including containers and + // canonical peers of the typed values (2 and 2.0, 1.5 as a decimal). + auto encoded_roots = ColumnVariantV2::create(); + for (const char* json : + {"null", "true", "1", "2", "2.0", "1.5", R"("a")", R"("2")", "[2]", R"({"k":2})"}) { + insert_encoded_field(*encoded_roots, encode_json(json)); + } + ASSERT_FALSE(encoded_roots->is_typed()); + + auto check = [&](const ColumnVariantV2::MutablePtr& typed) { + auto encoded = encoded_copy_of(*typed); + const auto* original = &typed->typed_column(); + for (size_t row = 0; row < typed->size(); ++row) { + for (size_t root = 0; root < encoded_roots->size(); ++root) { + const int expected = encoded->compare_at(row, root, *encoded_roots, 1); + EXPECT_EQ(typed->compare_at(row, root, *encoded_roots, 1), expected) + << "row=" << row << " root=" << root; + EXPECT_EQ(encoded_roots->compare_at(root, row, *typed, 1), -expected) + << "row=" << row << " root=" << root; + } + } + for (size_t root = 0; root < encoded_roots->size(); ++root) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(typed->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(typed->size(), 0); + typed->compare_internal(root, *encoded_roots, 1, direction, typed_cmp, + typed_filter.data()); + encoded->compare_internal(root, *encoded_roots, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } + // Comparing must not change the state of the typed column. + EXPECT_TRUE(typed->is_typed()); + EXPECT_EQ(&typed->typed_column(), original); + }; + constexpr std::array INTEGERS {1, 2, 3, 0}; + constexpr std::array INTEGER_NULLS {0, 0, 0, 1}; + check(typed_int32(INTEGERS, INTEGER_NULLS)); + check(ColumnVariantV2::create_typed( + nullable_fixed({1.5, 2.0, std::numeric_limits::quiet_NaN()}, + {0, 0, 0}), + std::make_shared())); + check(ColumnVariantV2::create_typed(nullable_decimal( + 1, {Decimal64 {15}, Decimal64 {20}}, {0, 0}), + std::make_shared(18, 1))); + constexpr std::array STRINGS {"a", "2", "b"}; + constexpr std::array STRING_NULLS {0, 0, 0}; + check(typed_strings(STRINGS, STRING_NULLS)); +} + +TEST(ColumnVariantV2Test, ShreddedComparisonMatchesEncodedAndStaysShredded) { + auto encoded = ColumnVariantV2::create(); + for (const char* json : {"2", "1.5", R"("a")", R"({"k":1})", "null"}) { + insert_encoded_field(*encoded, encode_json(json)); + } + auto materialized_calls = std::make_shared(0); + auto shredded = ColumnVariantV2::create_shredded( + std::make_shared(encoded->get_ptr(), materialized_calls)); + ASSERT_TRUE(shredded->is_shredded()); + constexpr std::array INTEGERS {1, 2, 3}; + constexpr std::array NOT_NULL {0, 0, 0}; + auto typed = typed_int32(INTEGERS, NOT_NULL); + auto typed_encoded = encoded_copy_of(*typed); + + for (size_t left = 0; left < encoded->size(); ++left) { + for (size_t right = 0; right < encoded->size(); ++right) { + const int expected = encoded->compare_at(left, right, *encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *shredded, 1), expected); + EXPECT_EQ(shredded->compare_at(left, right, *encoded, 1), expected); + EXPECT_EQ(encoded->compare_at(left, right, *shredded, 1), expected); + } + for (size_t right = 0; right < typed->size(); ++right) { + const int expected = encoded->compare_at(left, right, *typed_encoded, 1); + EXPECT_EQ(shredded->compare_at(left, right, *typed, 1), expected); + EXPECT_EQ(typed->compare_at(right, left, *shredded, 1), -expected); + } + } + // Comparison reads the rows the state materializes (a real state caches them); the column + // itself stays shredded, and the typed peer stays typed. + EXPECT_GT(*materialized_calls, 0); + EXPECT_TRUE(shredded->is_shredded()); + EXPECT_TRUE(typed->is_typed()); +} + +TEST(ColumnVariantV2Test, TypedNaNOrderingIgnoresNullDirectionHint) { + auto typed = ColumnVariantV2::create_typed( + nullable_fixed({1.0, std::numeric_limits::quiet_NaN()}, {0, 0}), + std::make_shared()); + auto encoded = encoded_copy_of(*typed); + for (int hint : {-1, 1}) { + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + EXPECT_EQ(typed->compare_at(left, right, *typed, hint), + encoded->compare_at(left, right, *encoded, hint)); + } + } + } +} + +TEST(ColumnVariantV2Test, TypedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { + constexpr std::array VALUES {2, 99, 1, 3}; + constexpr std::array NULLS {0, 1, 0, 0}; auto typed = typed_int32(VALUES, NULLS); expect_not_implemented([&] { static_cast(typed->get_data_at(0)); }, "intentionally unsupported"); + auto encoded = encoded_copy_of(*typed); + for (size_t left = 0; left < typed->size(); ++left) { + for (size_t right = 0; right < typed->size(); ++right) { + for (int nan_direction_hint : {-1, 1}) { + EXPECT_EQ(typed->compare_at(left, right, *typed, nan_direction_hint), + encoded->compare_at(left, right, *encoded, nan_direction_hint)); + } + } + } + for (size_t right = 0; right < typed->size(); ++right) { + for (int direction : {-1, 1}) { + std::vector typed_cmp(typed->size(), 0); + std::vector encoded_cmp(encoded->size(), 0); + IColumn::Filter typed_filter(typed->size(), 0); + IColumn::Filter encoded_filter(encoded->size(), 0); + typed->compare_internal(right, *typed, 1, direction, typed_cmp, typed_filter.data()); + encoded->compare_internal(right, *encoded, 1, direction, encoded_cmp, + encoded_filter.data()); + EXPECT_EQ(typed_cmp, encoded_cmp); + EXPECT_EQ(typed_filter, encoded_filter); + } + } HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { typed->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); + typed->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {1, 2, 0, 3})); + typed->get_permutation(true, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {3, 0})); expect_not_implemented([&] { typed->replace_column_data(*typed, 0); }, "intentionally unsupported"); EXPECT_TRUE(typed->is_typed()); } +TEST(ColumnVariantV2Test, PermutationLimitSwitchesBetweenPartialAndFullSort) { + // get_permutation only uses std::partial_sort while the limit stays small relative to the row + // count, the threshold ColumnVector applies as well; a larger limit falls back to the full + // sort. Both branches must order the leading rows the same way. + constexpr std::array VALUES {9, 3, 14, 7, 1, 12, 5, 15, 2, 11, 6, 13, 4, 10, 8, 0}; + constexpr std::array NULLS {}; + auto typed = typed_int32(VALUES, NULLS); + + HybridSorter sorter; + IColumn::Permutation result; + // limit 2 of 16 rows sits exactly on the threshold and still takes the partial_sort branch. + // Pin the whole prefix that branch must produce rather than only its first entry: the values 0 + // and 1 live at rows 15 and 4. The two branches cannot be told apart through the public API, so + // this pins the contract both of them owe, not the branch itself. + typed->get_permutation(false, 2, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.begin() + 2), (std::vector {15, 4})); + + // limit 12 of 16 rows exceeds the threshold, so the whole permutation comes out ordered. + typed->get_permutation(false, 12, 0, sorter, result); + ASSERT_EQ(result.size(), VALUES.size()); + for (size_t index = 1; index < result.size(); ++index) { + EXPECT_LT(typed->compare_at(result[index - 1], result[index], *typed, 1), 0); + } +} + TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { auto encoded = ColumnVariantV2::create(); insert_encoded_field(*encoded, encode_json("1")); @@ -3430,19 +3884,90 @@ TEST(ColumnVariantV2Test, ReplaceNullPayloadsWithCanonicalDefault) { EXPECT_EQ(json_at(*typed, 2), "{}"); } -TEST(ColumnVariantV2Test, EncodedPhysicalAndOrderingInterfacesStayUnsupported) { +TEST(ColumnVariantV2Test, EncodedPhysicalInterfacesStayUnsupportedAndOrderingWorks) { auto column = ColumnVariantV2::create(); - auto source = ColumnVariantV2::create(); - insert_encoded_field(*source, encode_json("1")); + insert_encoded_field(*column, encode_json(R"("text")")); + insert_encoded_field(*column, encode_json("2")); + insert_encoded_field(*column, encode_json("1.5")); + insert_encoded_field(*column, encode_json("true")); expect_not_implemented([&] { static_cast(column->get_data_at(0)); }, "intentionally unsupported"); HybridSorter sorter; IColumn::Permutation result; - expect_not_implemented([&] { column->get_permutation(false, 0, 0, sorter, result); }, - "intentionally unsupported"); - expect_not_implemented([&] { column->replace_column_data(*source, 0); }, + column->get_permutation(false, 0, 0, sorter, result); + EXPECT_EQ(std::vector(result.begin(), result.end()), (std::vector {3, 2, 1, 0})); + expect_not_implemented([&] { column->replace_column_data(*column, 0); }, "intentionally unsupported"); } +TEST(ColumnVariantV2Test, NullableMultiKeyOrderingWorks) { + auto variant = ColumnVariantV2::create(); + insert_encoded_field(*variant, encode_json("2")); + insert_encoded_field(*variant, encode_json("1")); + insert_encoded_field(*variant, encode_json("1")); + auto nullable = ColumnNullable::create(std::move(variant), ColumnUInt8::create(3, 0)); + + auto tie = ColumnInt32::create(); + tie->insert_value(0); + tie->insert_value(2); + tie->insert_value(1); + auto ids = ColumnInt32::create(); + ids->insert_value(1); + ids->insert_value(2); + ids->insert_value(3); + + Block source; + source.insert({std::move(nullable), make_nullable(std::make_shared()), "v"}); + source.insert({std::move(tie), std::make_shared(), "tie"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, 1, 1); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + + const auto& sorted_ids = assert_cast(*sorted.get_by_position(2).column); + EXPECT_EQ(sorted_ids.get_data()[0], 3); + EXPECT_EQ(sorted_ids.get_data()[1], 2); + EXPECT_EQ(sorted_ids.get_data()[2], 1); +} + +TEST(ColumnVariantV2Test, NullableTypedSortMatchesEncoded) { + constexpr std::array VALUES {3, 0, 1, 7, 1}; + // Row 1 is a Variant null inside the value; row 2 is a SQL NULL in the outer column. + constexpr std::array VARIANT_NULLS {0, 1, 0, 0, 0}; + constexpr std::array SQL_NULLS {0, 0, 1, 0, 0}; + auto sorted_ids = [&](const ColumnPtr& variant, int direction, int nulls_direction) { + auto nulls = ColumnUInt8::create(); + nulls->get_data().assign(SQL_NULLS.begin(), SQL_NULLS.end()); + auto ids = ColumnInt32::create(); + for (int32_t id = 0; id < static_cast(VALUES.size()); ++id) { + ids->insert_value(id); + } + Block source; + source.insert({ColumnNullable::create(variant, std::move(nulls)), + make_nullable(std::make_shared()), "v"}); + source.insert({std::move(ids), std::make_shared(), "id"}); + SortDescription description; + description.emplace_back(0, direction, nulls_direction); + description.emplace_back(1, 1, 1); + Block sorted = source.clone_empty(); + HybridSorter sorter; + sort_block(source, sorted, description, sorter); + const auto& output = assert_cast(*sorted.get_by_position(1).column); + return std::vector(output.get_data().begin(), output.get_data().end()); + }; + auto typed = typed_int32(VALUES, VARIANT_NULLS); + auto encoded = encoded_copy_of(*typed); + for (int direction : {-1, 1}) { + for (int nulls_direction : {-1, 1}) { + EXPECT_EQ(sorted_ids(typed->get_ptr(), direction, nulls_direction), + sorted_ids(encoded->get_ptr(), direction, nulls_direction)) + << "direction=" << direction << " nulls_direction=" << nulls_direction; + } + } +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp index 1b5dddf8e9e925..5cecf9d9cefa00 100644 --- a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp @@ -15,27 +15,35 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include +#include #include #include #include #include +#include #include "core/arena.h" #include "core/assert_cast.h" +#include "core/column/column_array.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_number.h" +#include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_variant_v2_serde.h" #include "core/string_buffer.hpp" +#include "core/value/variant/variant_batch_builder.h" +#include "exprs/function/parse/variant_string_parse.h" #include "util/jsonb_document.h" +#include "util/jsonb_utils.h" #include "util/jsonb_writer.h" namespace doris { @@ -103,6 +111,33 @@ std::string json_at(const DataTypeVariantV2SerDe& serde, const IColumn& column, return output->get_data_at(0).to_string(); } +ColumnVariantV2::MutablePtr encoded_json(std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (std::string_view row : rows) { + encoder.add_json({row.data(), row.size()}); + } + VariantBatchBuilder block = encoder.finish_batch(); + auto result = ColumnVariantV2::create(); + result->insert_encoded_batch(block); + return result; +} + +std::vector jsonb_vector_as_json( + const DataTypeSerDe& serde, const IColumn& column, + const DataTypeSerDe::FormatOptions& options = DataTypeSerDe::FormatOptions {}) { + auto output = ColumnString::create(); + const Status status = serde.serialize_column_to_jsonb_vector(column, *output, options); + EXPECT_TRUE(status.ok()) << status; + std::vector rows; + for (size_t row = 0; row < output->size(); ++row) { + const StringRef bytes = output->get_data_at(row); + rows.push_back(JsonbToJson::jsonb_to_json_string(bytes.data, bytes.size)); + } + return rows; +} + std::string object_with_value(const std::function& write_value) { JsonbWriter writer; EXPECT_TRUE(writer.writeStartObject()); @@ -114,6 +149,51 @@ std::string object_with_value(const std::function& write_val } // namespace +TEST(DataTypeVariantV2SerdeJsonbTest, VectorSkipsMaskedTypedNonFiniteValues) { + auto values = ColumnFloat64::create(); + values->insert_value(1.5); + values->insert_value(std::numeric_limits::quiet_NaN()); + values->insert_value(2.5); + auto column = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(values), ColumnUInt8::create(3, 0)), + std::make_shared()); + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + auto output = ColumnString::create(); + const std::array null_map {0, 1, 0}; + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), "1.5"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), "2.5"); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, VectorPreservesMaskedEncodedAndConstantRowPositions) { + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions options; + const std::array null_map {0, 1, 0}; + auto encoded = encoded_json({"1", "null", "2"}); + auto constant = ColumnConst::create(typed_int(42), 3); + for (const IColumn* column : {static_cast(encoded.get()), + static_cast(constant.get())}) { + auto output = ColumnString::create(); + ASSERT_TRUE( + serde.serialize_column_to_jsonb_vector(*column, *output, options, null_map.data()) + .ok()); + ASSERT_EQ(output->size(), 3); + EXPECT_EQ(output->get_data_at(1).size, 0); + const auto first = output->get_data_at(0); + const auto last = output->get_data_at(2); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(first.data, first.size), + column == encoded.get() ? "1" : "42"); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(last.data, last.size), + column == encoded.get() ? "2" : "42"); + } +} + TEST(DataTypeVariantV2SerdeJsonbTest, EncodedAndTypedWriteIdenticalBinaryDocumentsAndRoundTrip) { DataTypeVariantV2SerDe serde; auto typed = typed_int(42); @@ -243,4 +323,80 @@ TEST(DataTypeVariantV2SerdeJsonbTest, InvalidRowDoesNotTouchResultWriter) { EXPECT_EQ(writer.getOutput()->getSize(), before); } +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbWritesTheJsonDocument) { + DataTypeVariantV2SerDe serde; + auto encoded = encoded_json({R"({"b":[true,null,"x"],"a":{"c":1.5}})", "42", R"("s")", "null"}); + EXPECT_EQ(jsonb_vector_as_json(serde, *encoded), + (std::vector {R"({"a":{"c":1.5},"b":[true,null,"x"]})", "42", R"("s")", + "null"})); + + // Typed and encoded columns write the same bytes, including a Variant null. + for (const bool is_null : {false, true}) { + auto typed = typed_int(42, is_null); + ColumnPtr encoded_int = encoded_copy(*typed); + auto typed_output = ColumnString::create(); + auto encoded_output = ColumnString::create(); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*typed, *typed_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_TRUE(serde.serialize_column_to_jsonb_vector(*encoded_int, *encoded_output, + DataTypeSerDe::FormatOptions {}) + .ok()); + ASSERT_EQ(typed_output->size(), 1); + EXPECT_EQ(typed_output->get_data_at(0), encoded_output->get_data_at(0)); + EXPECT_EQ(jsonb_vector_as_json(serde, *typed)[0], is_null ? "null" : "42"); + } + + ColumnPtr constant = ColumnConst::create(encoded_json({R"({"k":"v"})"})->get_ptr(), 2); + EXPECT_EQ(jsonb_vector_as_json(serde, *constant), + (std::vector {R"({"k":"v"})", R"({"k":"v"})"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbNestsVariantInArray) { + auto values = encoded_json({R"({"a":[1,2]})", "7", R"("x")"}); + auto value_nulls = ColumnUInt8::create(); + for (const uint8_t is_null : {0, 1, 0}) { + value_nulls->insert_value(is_null); + } + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(2); + offsets->insert_value(3); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), std::move(value_nulls)), std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array), + (std::vector {R"([{"a":[1,2]},null])", R"(["x"])"})); +} + +TEST(DataTypeVariantV2SerdeJsonbTest, SerializeColumnToJsonbUsesTheSessionTimeZone) { + VariantBatchBuilder builder; + auto row = builder.begin_row(); + row.add_timestamp_micros(0, true); + row.finish(); + VariantBatchBuilder block = builder.finish_batch(); + auto values = ColumnVariantV2::create(); + values->insert_encoded_batch(block); + + DataTypeVariantV2SerDe serde; + const cctz::time_zone shanghai = cctz::fixed_time_zone(cctz::seconds(8 * 3600)); + DataTypeSerDe::FormatOptions options; + options.timezone = &shanghai; + EXPECT_EQ(jsonb_vector_as_json(serde, *values, options), + (std::vector {R"("1970-01-01 08:00:00.000000+08:00")"})); + EXPECT_EQ(jsonb_vector_as_json(serde, *values), + (std::vector {R"("1970-01-01 00:00:00.000000+00:00")"})); + + // A nested Variant element receives the same time zone. + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto array = ColumnArray::create( + ColumnNullable::create(std::move(values), ColumnUInt8::create(1, 0)), + std::move(offsets)); + DataTypeArraySerDe array_serde( + std::make_shared(std::make_shared())); + EXPECT_EQ(jsonb_vector_as_json(array_serde, *array, options), + (std::vector {R"(["1970-01-01 08:00:00.000000+08:00"])"})); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp index 2972e7023da4ae..b8b1c598ae0261 100644 --- a/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_write_to_jsonb_test.cpp @@ -53,7 +53,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt32::create(); @@ -65,7 +66,8 @@ TEST(DataTypeWritToJsonb, test_number) { { auto data = ColumnHelper::create_column_with_name({1, 2, 3, 4, 5}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4"); auto data_to = ColumnInt64::create(); EXPECT_TRUE(data.type->get_serde()->deserialize_column_from_jsonb( @@ -77,7 +79,8 @@ TEST(DataTypeWritToJsonb, test_number) { auto data = ColumnHelper::create_column_with_name( {1.1F, 2.2F, 3.3F, 4.4F, 5.5F}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "4.4"); auto data_to = ColumnFloat32::create(); @@ -92,7 +95,8 @@ TEST(DataTypeWritToJsonb, test_number) { col->insert_value(val); auto type = std::make_shared(18, 2); JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col, 0, writer, + DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "123.45"); auto data_to = ColumnDecimal128V3::create(0, 2); @@ -109,7 +113,8 @@ TEST(DataTypeWritToJsonb, test_string) { auto data = ColumnHelper::create_column_with_name( {"hello", "world", "doris", "vectorized", "test"}); JsonbWriter writer; - EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb(*data.column, 3, writer)); + EXPECT_TRUE(data.type->get_serde()->serialize_column_to_jsonb( + *data.column, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "\"vectorized\""); auto data_to = ColumnString::create(); @@ -137,7 +142,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[1,2,3]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -148,7 +154,8 @@ TEST(DataTypeWritToJsonb, test_array) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_array, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_array, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "[null,5]"); auto col_array_to = col_array->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -182,7 +189,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 0, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 0, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":1,\"string_col\":\"hello\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -194,7 +202,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 1, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 1, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":2,\"string_col\":\"world\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -206,7 +215,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 2, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 2, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":3,\"string_col\":null}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -217,7 +227,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 3, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 3, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":null,\"string_col\":\"vectorized\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( @@ -229,7 +240,8 @@ TEST(DataTypeWritToJsonb, test_struct) { { JsonbWriter writer; - EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb(*col_struct, 4, writer)); + EXPECT_TRUE(type->get_serde()->serialize_column_to_jsonb( + *col_struct, 4, writer, DataTypeSerDe::FormatOptions {})); EXPECT_EQ(to_string(writer), "{\"int_col\":5,\"string_col\":\"test\"}"); auto col_struct_to = col_struct->clone_empty(); EXPECT_TRUE(type->get_serde()->deserialize_column_from_jsonb( diff --git a/be/test/expected_result/vec/columns/column_variant_byte_size.out b/be/test/expected_result/vec/columns/column_variant_byte_size.out index 7e37b506ec5714..3ac94e60f77efd 100644 --- a/be/test/expected_result/vec/columns/column_variant_byte_size.out +++ b/be/test/expected_result/vec/columns/column_variant_byte_size.out @@ -1,2 +1,2 @@ -column: variant with byte_size: 11295900 -11295900 \ No newline at end of file +column: variant with byte_size: 11295888 +11295888 \ No newline at end of file diff --git a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp index 54bedaa5c52cf0..5ed330f46425d1 100644 --- a/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp +++ b/be/test/exprs/function/cast/cast_variant_v2_to_test.cpp @@ -95,6 +95,43 @@ TEST(CastVariantV2ToTest, UnsupportedSourceReturnsAnError) { EXPECT_EQ(cast.column.get(), cast.initial_result.get()); } +TEST(CastVariantV2ToTest, InvalidUtf8StringSourceReturnsAnError) { + const std::array source_types { + std::make_shared(), std::make_shared(16, TYPE_CHAR), + std::make_shared(64, TYPE_VARCHAR)}; + for (const DataTypePtr& source_type : source_types) { + auto source = ColumnString::create(); + source->insert_data("ok", 2); + source->insert_data("\xFF", 1); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) + << source_type->get_name() << ": " << cast.status; + + // A row that the cast turns into NULL never becomes a Variant string. + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << source_type->get_name() << ": " << null_row.status; + } +} + +TEST(CastVariantV2ToTest, InvalidDateSourceReturnsAnError) { + // The same check covers every scalar Variant cannot hold, not only strings. + auto source = ColumnDateV2::create(); + DateV2Value valid; + valid.unchecked_set_time(2024, 1, 2, 0, 0, 0); + DateV2Value invalid; + ASSERT_FALSE(invalid.is_valid_date()); + source->insert_value(valid); + source->insert_value(invalid); + const auto source_type = std::make_shared(); + CastResult cast = execute_to_variant(source->get_ptr(), source_type); + EXPECT_TRUE(cast.status.is()) << cast.status; + + const std::array null_map {0, 1}; + CastResult null_row = execute_to_variant(source->get_ptr(), source_type, null_map.data()); + EXPECT_TRUE(null_row.status.ok()) << null_row.status; +} + TEST(CastVariantV2ToTest, Decimal256ReturnsAnErrorInsteadOfStringifying) { auto type = std::make_shared(76, 2); auto source = ColumnDecimal256::create(0, 2); diff --git a/be/test/exprs/variant_v2_execution_test.cpp b/be/test/exprs/variant_v2_execution_test.cpp index 3aed252297870d..36fd2e1bec95df 100644 --- a/be/test/exprs/variant_v2_execution_test.cpp +++ b/be/test/exprs/variant_v2_execution_test.cpp @@ -19,8 +19,12 @@ #include "core/column/column_variant.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" +#include "exprs/function/functions_comparison.h" +#include "exprs/function/parse/variant_string_parse.h" namespace doris { @@ -32,4 +36,79 @@ TEST(VariantV2ExecutionTest, ExecutionTypeSelectsPhysicalColumn) { EXPECT_NE(check_and_get_column(*compute_v2.create_column()), nullptr); } +TEST(VariantV2ExecutionTest, TypePropertiesDoNotAffectEquality) { + // Execution compares and hashes Variant values canonically; storage properties such as the + // maximum subcolumn count and doc mode must not make two Variant types different. + const DataTypePtr plain = std::make_shared(2048, false); + const DataTypePtr limited = std::make_shared(10, false); + const DataTypePtr doc_mode = std::make_shared(0, true); + EXPECT_TRUE(plain->equals(*limited)); + EXPECT_TRUE(limited->equals(*doc_mode)); + EXPECT_TRUE(make_nullable(plain)->equals(*make_nullable(doc_mode))); + EXPECT_FALSE(plain->equals(DataTypeInt32())); +} + +TEST(VariantV2ExecutionTest, CanonicalComparison) { + auto parse = [](std::initializer_list rows) { + JsonStringToVariantEncoder encoder({.max_json_key_length = 255, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (auto json : rows) { + encoder.add_json({json.data(), json.size()}); + } + auto column = ColumnVariantV2::create(); + column->insert_encoded_batch(encoder.finish_batch()); + return column; + }; + auto left = parse({"1", "1", "null", R"({"a":1,"b":[2,null]})", "[1,2]", "true"}); + auto right = parse({"1.0", R"("1")", "null", R"({"b":[2.0,null],"a":1.0})", "[2,1]", "1"}); + auto type = std::make_shared(); + Block block {{left->get_ptr(), type, "left"}, + {right->get_ptr(), type, "right"}, + {nullptr, std::make_shared(), "result"}}; + FunctionComparison equals; + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 0, 1, 1, 0, 0})); + FunctionComparison not_equals; + ASSERT_TRUE(not_equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + FunctionComparison less; + ASSERT_TRUE(less.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {0, 1, 0, 0, 1, 1})); + + auto one = parse({"1.0"}); + block.get_by_position(1).column = ColumnConst::create(std::move(one), left->size()); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {0, 1}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); + ASSERT_TRUE(equals.execute_impl(nullptr, block, {1, 0}, 2, left->size()).ok()); + EXPECT_EQ(assert_cast(*block.get_by_position(2).column).get_data(), + (ColumnUInt8::Container {1, 1, 0, 0, 0, 0})); +} + +TEST(VariantV2ExecutionTest, TypedEquality) { + auto numbers = ColumnInt32::create(); + numbers->get_data() = {1, 2, 3}; + auto nulls = ColumnUInt8::create(); + nulls->get_data() = {0, 0, 1}; + auto typed = ColumnVariantV2::create_typed( + ColumnNullable::create(std::move(numbers), std::move(nulls)), + std::make_shared()); + auto encoded = ColumnVariantV2::create(); + encoded->insert_range_from(*typed, 0, typed->size()); + encoded->ensure_encoded(); + ASSERT_TRUE(typed->is_typed()); + ASSERT_FALSE(encoded->is_typed()); + for (const auto& left : {typed->get_ptr(), encoded->get_ptr()}) { + for (const auto& right : {typed->get_ptr(), encoded->get_ptr()}) { + for (size_t row = 0; row < left->size(); ++row) { + EXPECT_EQ(left->compare_at(row, row, *right, 1), 0); + } + } + } +} + } // namespace doris diff --git a/be/test/util/variant/variant_canonical_test.cpp b/be/test/util/variant/variant_canonical_test.cpp index 42662c7262cd1b..83f2189494f16c 100644 --- a/be/test/util/variant/variant_canonical_test.cpp +++ b/be/test/util/variant/variant_canonical_test.cpp @@ -384,6 +384,7 @@ void expect_scalar_equivalent(const VariantScalarRef& scalar_ref, const OwnedVal EXPECT_EQ(scalar_hashes.crc, encoded_hashes.crc); EXPECT_EQ(scalar_hashes.crc32c, encoded_hashes.crc32c); EXPECT_EQ(arena(scalar_ref), arena(encoded_value.ref())); + EXPECT_EQ(canonical_compare(scalar_ref, encoded_value.ref()), 0); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- exhaustive typed/encoded parity matrix. @@ -548,6 +549,119 @@ TEST(VariantCanonicalTest, NumericEquivalenceMatrix) { VariantPrimitiveId::DOUBLE); } +TEST(VariantCanonicalTest, ComparisonPreservesEqualityAndNumericOrder) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_FALSE(canonical_equals(left.ref(), right.ref())); + }; + const auto expect_equal = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_EQ(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_EQ(canonical_compare(right.ref(), left.ref()), 0); + EXPECT_TRUE(canonical_equals(left.ref(), right.ref())); + EXPECT_EQ(hashes(left.ref()).sip, hashes(right.ref()).sip); + }; + + expect_order(decimal_value(15, 1), integer_value(100, 1)); + expect_order(decimal_value(1, 1), double_value(0.1)); + expect_order(double_value(std::nextafter(1.0, 0.0)), integer_value(1, 1)); + expect_order(integer_value(1, 1), double_value(std::nextafter(1.0, 2.0))); + expect_order(decimal_value(15, 1), double_value(1.5)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-999999999999999999LL, 1)); + expect_order(double_value(std::numeric_limits::infinity()), + double_value(std::numeric_limits::quiet_NaN())); + + expect_equal(integer_value(42, 1), decimal_value(4200, 2)); + expect_equal(integer_value(42, 1), double_value(42.0)); + expect_equal(double_bits(0x7FF0000000000001ULL), double_bits(0xFFFABCDE12345678ULL)); +} + +TEST(VariantCanonicalTest, ContainerComparisonIsLexicographicAndTransitive) { + const OwnedValue array_a = array_value({integer_value(1, 1).value}); + const OwnedValue array_b = array_value({integer_value(1, 1).value, integer_value(2, 1).value}); + const OwnedValue array_c = array_value({integer_value(1, 1).value, integer_value(3, 1).value}); + EXPECT_LT(canonical_compare(array_a.ref(), array_b.ref()), 0); + EXPECT_LT(canonical_compare(array_b.ref(), array_c.ref()), 0); + EXPECT_LT(canonical_compare(array_a.ref(), array_c.ref()), 0); + + const OwnedValue object_a = object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0}); + const OwnedValue object_b = object_value({"a"}, true, {0}, {integer_value(2, 1).value}, {0}); + const OwnedValue object_c = object_value({"b"}, true, {0}, {integer_value(0, 1).value}, {0}); + EXPECT_LT(canonical_compare(object_a.ref(), object_b.ref()), 0); + EXPECT_LT(canonical_compare(object_b.ref(), object_c.ref()), 0); + EXPECT_LT(canonical_compare(object_a.ref(), object_c.ref()), 0); + EXPECT_GT(canonical_compare(object_c.ref(), object_a.ref()), 0); +} + +TEST(VariantCanonicalTest, ComparisonBoundariesAndCrossKindTransitivity) { + const auto expect_order = [](const OwnedValue& left, const OwnedValue& right) { + EXPECT_LT(canonical_compare(left.ref(), right.ref()), 0); + EXPECT_GT(canonical_compare(right.ref(), left.ref()), 0); + }; + __int128 max_38_digits = 1; + for (int digit = 0; digit < 38; ++digit) { + max_38_digits *= 10; + } + max_38_digits -= 1; + + // Scaling 38-digit magnitudes by 10^38 needs the 256-bit intermediate. + expect_order(decimal_value(max_38_digits, 38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(-max_38_digits, 0), decimal_value(-max_38_digits, 38)); + // Subnormal and large doubles against exact values at the decimal limits. + expect_order(double_value(std::numeric_limits::denorm_min()), decimal_value(1, 38)); + expect_order(double_value(1e38), decimal_value(max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), double_value(1.7e38)); + expect_order(double_value(-std::numeric_limits::infinity()), + decimal_value(-max_38_digits, 0)); + expect_order(decimal_value(max_38_digits, 0), + double_value(std::numeric_limits::infinity())); + + // Numeric value first, canonical kind as the tie-breaker, must stay transitive. + const std::array chain {decimal_value(15, 1), double_value(1.5), + integer_value(2, 1), decimal_value(25, 1)}; + for (size_t left = 0; left < chain.size(); ++left) { + for (size_t right = left + 1; right < chain.size(); ++right) { + expect_order(chain[left], chain[right]); + } + } + // Numbers order before strings and containers. + expect_order(double_value(std::numeric_limits::quiet_NaN()), string_value("0")); + expect_order(string_value("0"), array_value({})); +} + +TEST(VariantCanonicalTest, ScalarAgainstEncodedRootMatchesEncodedComparison) { + // The mixed overload must order a scalar exactly as its encoded bytes would be ordered, for + // every kind of encoded root including the containers a scalar can never equal. + const std::string text = "b"; + const std::array scalars { + VariantScalarRef::null_value(), + VariantScalarRef::boolean(true), + VariantScalarRef::integer(2), + VariantScalarRef::decimal(15, 1), + VariantScalarRef::float64(1.5), + VariantScalarRef::float64(std::numeric_limits::quiet_NaN()), + VariantScalarRef::string(StringRef(text)), + VariantScalarRef::date(19000)}; + const std::array roots { + scalar(primitive(VariantPrimitiveId::NULL_VALUE)), + integer_value(2, 1), + decimal_value(15, 1), + double_value(1.5), + double_value(2.5), + string_value("a"), + string_value("b"), + array_value({integer_value(1, 1).value}), + object_value({"a"}, true, {0}, {integer_value(1, 1).value}, {0})}; + for (const VariantScalarRef& value : scalars) { + const OwnedValue encoded = scalar(physical(value)); + for (const OwnedValue& root : roots) { + EXPECT_EQ(canonical_compare(value, root.ref()), + canonical_compare(encoded.ref(), root.ref())); + } + } +} + TEST(VariantCanonicalTest, PrimitiveTypeClasses) { const OwnedValue null_a = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); const OwnedValue null_b = scalar(primitive(VariantPrimitiveId::NULL_VALUE)); @@ -881,6 +995,11 @@ TEST(VariantCanonicalTest, RandomizedEqualsHashAndArenaProperties) { auto [left, right] = random_pair(random, index, expected_equal); const bool equal = canonical_equals(left.ref(), right.ref()); EXPECT_EQ(equal, expected_equal); + const int comparison = canonical_compare(left.ref(), right.ref()); + const int reverse_comparison = canonical_compare(right.ref(), left.ref()); + EXPECT_EQ(comparison == 0, equal); + EXPECT_EQ((comparison > 0) - (comparison < 0), + -((reverse_comparison > 0) - (reverse_comparison < 0))); const std::string left_arena = arena(left.ref()); const std::string right_arena = arena(right.ref()); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index fc214943541985..6c9badeaa375b6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -285,7 +285,8 @@ public abstract class Type { variantSubTypes.add(DECIMAL256); variantSubTypes.add(DATEV2); variantSubTypes.add(DATETIMEV2); - variantSubTypes.add(TIMESTAMP_TZ); + // TIMESTAMP_TZ is deliberately absent. This list is only checked when a column or + // function type is defined, so tables that already declare a TIMESTAMPTZ path keep working. variantSubTypes.add(IPV4); variantSubTypes.add(IPV6); variantSubTypes.add(STRING); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index 032894cf8602a6..effbd78ee19432 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -142,7 +142,9 @@ public static PushDownContext createPushDownContextForHashJoin(Expression srcExp } public boolean isValid() { - return finalTarget != null && probeSlot != null; + return finalTarget != null && probeSlot != null + && !srcExpr.getDataType().isVariantType() + && !probeExpr.getDataType().isVariantType(); } public PushDownContext withNewProbeExpression(Expression newProbe) { @@ -185,6 +187,9 @@ public Boolean visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapS @Override public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) { + if (ctx.srcExpr.getDataType().isVariantType() || ctx.probeExpr.getDataType().isVariantType()) { + return false; + } if (scan instanceof PhysicalSchemaScan) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 50826ece82ae81..7d23671126f6f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -46,7 +46,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; -import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VariantType; import com.google.common.collect.Sets; @@ -149,7 +148,7 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { LogicalAggregate agg = (LogicalAggregate) plan; for (Expression groupBy : agg.getGroupByExpressions()) { if (groupBy.getDataType().isObjectType() - || isLegacyVariant(groupBy.getDataType()) + || VariantType.isLegacyVariant(groupBy.getDataType()) || groupBy.getDataType().isVarBinaryType()) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } @@ -157,14 +156,16 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } else if (plan instanceof LogicalSort) { LogicalSort sort = (LogicalSort) plan; for (OrderKey orderKey : sort.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getExpr().getDataType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } } else if (plan instanceof LogicalTopN) { LogicalTopN topN = (LogicalTopN) plan; for (OrderKey orderKey : topN.getOrderKeys()) { - if (orderKey.getExpr().getDataType().isObjectOrVariantType()) { + if (orderKey.getExpr().getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getExpr().getDataType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } @@ -174,24 +175,26 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { return; } WindowExpression windowExpression = (WindowExpression) ((Alias) a).child(); - if (windowExpression.getOrderKeys().stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + if (windowExpression.getOrderKeys().stream().anyMatch(orderKey -> + orderKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } - if (windowExpression.getPartitionKeys().stream().anyMatch(( - partitionKey -> partitionKey.getDataType().isObjectOrVariantType()))) { + if (windowExpression.getPartitionKeys().stream().anyMatch(partitionKey -> + partitionKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(partitionKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } }); } else if (plan instanceof LogicalSetOperation && ((LogicalSetOperation) plan).getQualifier() == Qualifier.DISTINCT) { - if (plan.getOutput().stream().anyMatch(output -> isLegacyVariant(output.getDataType()))) { + if (plan.getOutput().stream().anyMatch(output -> VariantType.isLegacyVariant(output.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } else if (plan instanceof LogicalJoin) { LogicalJoin join = (LogicalJoin) plan; for (Expression conjunct : join.getHashJoinConjuncts()) { - if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVariantType())) { + if (conjunct.anyMatch(e -> VariantType.isLegacyVariant(((Expression) e).getDataType()))) { throw new AnalysisException("variant type could not in join equal conditions: " + conjunct.toSql()); } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( @@ -199,7 +202,7 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } } for (Expression conjunct : join.getMarkJoinConjuncts()) { - if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVariantType())) { + if (conjunct.anyMatch(e -> VariantType.isLegacyVariant(((Expression) e).getDataType()))) { throw new AnalysisException("variant type could not in join equal conditions: " + conjunct.toSql()); } else if (conjunct.anyMatch(e -> ((Expression) e).getDataType().isVarBinaryType())) { throw new AnalysisException( @@ -209,10 +212,6 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { } } - private boolean isLegacyVariant(DataType dataType) { - return dataType instanceof VariantType && !((VariantType) dataType).isExecutionV2(); - } - private void checkMatchIsUsedCorrectly(Plan plan) { for (Expression expression : plan.getExpressions()) { if (expression instanceof Match) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java index 762896a240844b..9c1853d7a9911d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOrderByKey.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.types.VariantType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -81,8 +82,9 @@ private static Plan eliminateWindow(LogicalWindow window) { Alias alias = (Alias) expr; WindowExpression windowExpression = (WindowExpression) alias.child(); List orderExpressions = windowExpression.getOrderKeys(); - if (orderExpressions.stream().anyMatch(( - orderKey -> orderKey.getDataType().isObjectOrVariantType()))) { + if (orderExpressions.stream().anyMatch(orderKey -> + orderKey.getDataType().isObjectType() + || VariantType.isLegacyVariant(orderKey.getDataType()))) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } List orderKeys = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java index ce262e8610cb4e..18e99c3c7e7888 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java @@ -19,9 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.JsonType; import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.ComplexDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -44,11 +42,6 @@ static boolean isExplicitlyCastable(DataType signatureType, DataType realType) { /** isExplicitlyCastable */ static boolean isPrimitiveExplicitlyCastable(DataType signatureType, DataType realType) { - if (signatureType instanceof JsonType - && realType instanceof VariantType - && ((VariantType) realType).isExecutionV2()) { - return false; - } if (signatureType instanceof AnyDataType || signatureType instanceof FollowToAnyDataType || signatureType.isAssignableFrom(realType)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java index 6345f29f036c81..e53465515d898b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonHash.java @@ -19,7 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.NullOrIdenticalSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -35,7 +35,7 @@ * to_json convert type to json */ public class JsonHash extends ScalarFunction - implements UnaryExpression, NullOrIdenticalSignature, PropagateNullable { + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(JsonType.INSTANCE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java index a6e571c23231c6..db9830cea694c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VariantType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -96,7 +97,8 @@ public ToJson withChildren(List children) { @Override public List getSignatures() { DataType firstChildType = child(0).getDataType(); - if (firstChildType.isStructType() || firstChildType.isArrayType()) { + if (firstChildType.isStructType() || firstChildType.isArrayType() + || (firstChildType instanceof VariantType && ((VariantType) firstChildType).isExecutionV2())) { return ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE).args(firstChildType)); } if (firstChildType.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 0acc6340e916ef..d9ec97fb155064 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IntegralDivide; @@ -224,11 +225,8 @@ public static Optional implicitCast(DataType input, DataType expected) } return Optional.of(new StructType(newFields)); } else if (input instanceof VariantType - && ((VariantType) input).isExecutionV2() && expected instanceof JsonType) { - // JSON functions require users to make this representation change explicit. - return Optional.empty(); - } else if (input instanceof VariantType && (expected.isNumericType() || expected.isStringLikeType())) { - // variant could implicit cast to numric types and string like types + && (expected.isNumericType() || expected.isStringLikeType() || expected.isJsonType())) { + // variant could implicit cast to numric types, string like types and json return Optional.of(expected); } else { return implicitCastPrimitive(input, expected); @@ -1400,16 +1398,29 @@ public static Expression processComparisonPredicate(ComparisonPredicate comparis Expression left = comparisonPredicate.left(); Expression right = comparisonPredicate.right(); - boolean leftIsVariantV2 = left.getDataType() instanceof VariantType + boolean leftIsVariant = left.getDataType() instanceof VariantType && ((VariantType) left.getDataType()).isExecutionV2(); - boolean rightIsVariantV2 = right.getDataType() instanceof VariantType + boolean rightIsVariant = right.getDataType() instanceof VariantType && ((VariantType) right.getDataType()).isExecutionV2(); - boolean isDirectVariantSubpathScalarComparison = leftIsVariantV2 != rightIsVariantV2 - && ((leftIsVariantV2 && left instanceof ElementAt) - || (rightIsVariantV2 && right instanceof ElementAt)); - if ((leftIsVariantV2 || rightIsVariantV2) + // V2 equality is shared by scalar predicates and canonical hash join keys. Keep + // ordering and mixed Variant/scalar comparisons on their existing coercion paths. + if (leftIsVariant && rightIsVariant && comparisonPredicate instanceof EqualPredicate) { + return comparisonPredicate; + } + // A bare NULL has no type of its own, so it takes the Variant type of the other side and + // `v = NULL`, `v != NULL` and `v <=> NULL` become the Variant equality above. + if (comparisonPredicate instanceof EqualPredicate && leftIsVariant != rightIsVariant + && (leftIsVariant ? right : left).getDataType().isNullType()) { + DataType variantDataType = leftIsVariant ? left.getDataType() : right.getDataType(); + return comparisonPredicate.withChildren(castIfNotSameType(left, variantDataType), + castIfNotSameType(right, variantDataType)); + } + boolean isDirectVariantSubpathScalarComparison = leftIsVariant != rightIsVariant + && ((leftIsVariant && left instanceof ElementAt) + || (rightIsVariant && right instanceof ElementAt)); + if ((leftIsVariant || rightIsVariant) && !isDirectVariantSubpathScalarComparison) { - DataType variantDataType = leftIsVariantV2 + DataType variantDataType = leftIsVariant ? left.getDataType() : right.getDataType(); throw new AnalysisException("data type " + variantDataType + " could not used in ComparisonPredicate " + comparisonPredicate.toSql() diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index 0acf44f3c2ec7e..f7fb2205f7472e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -26,6 +26,14 @@ public class TypeTest { + @Test + public void testTimestampTzIsNotVariantSubtype() { + Assert.assertFalse(Type.VARIANT.supportSubType(ScalarType.createTimeStampTzType(6))); + Assert.assertFalse(Type.VARIANT.supportSubType( + new ArrayType(ScalarType.createTimeStampTzType(3), true))); + Assert.assertTrue(Type.VARIANT.supportSubType(ScalarType.createDatetimeV2Type(6))); + } + // ===================== ArrayType ===================== @Test public void testArrayOfArrayExactMatch() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java index b5beafa169356b..f326d74fb73325 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; @@ -276,6 +277,18 @@ private String evidence() { } } + @Test + public void testTargetVariantTypeParticipatesInTryParseEquality() { + StringLiteral json = new StringLiteral("{\"a\":1}"); + VariantType firstTarget = new VariantType(1); + VariantType secondTarget = new VariantType(2); + + TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); + TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); + Assertions.assertNotEquals(firstTryParse, secondTryParse); + Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); + } + @FunctionalInterface private interface LoadFixtureSupplier { LoadFixture get() throws Exception; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java index bd8a8af9bc3401..bc8d3c8ef0e54f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java @@ -29,11 +29,13 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; +import org.apache.doris.nereids.processor.post.RuntimeFilterPushDownVisitor.PushDownContext; import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterV2; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -57,6 +59,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.RuntimeFilterId; @@ -88,6 +91,10 @@ public void runBeforeAll() throws Exception { connectContext.getSessionVariable().setEnableRuntimeFilterPrune(false); connectContext.getSessionVariable().expandRuntimeFilterByInnerJoin = false; connectContext.getSessionVariable().setDisableJoinReorder(true); + createTable("CREATE TABLE variant_rf_a (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); + createTable("CREATE TABLE variant_rf_b (k INT, v VARIANT) DUPLICATE KEY(k) " + + "DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); } @Test @@ -592,4 +599,91 @@ public void testRuntimeFilterBlockByRecCte() { List filters = context.getNereidsRuntimeFilter(); Assertions.assertEquals(0, filters.size()); } + + @Test + public void testVariantJoinKeyDoesNotGenerateRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + // Control: the INT key of the same tables produces a runtime filter in this setup. + Assertions.assertFalse(getRuntimeFilters( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.k = b.k").get().isEmpty()); + // Scalar runtime filters cannot express Variant canonical equality. + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v = b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v <=> b.v", + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b ON a.v['id'] = b.v['id']", + "SELECT * FROM (SELECT CAST(k AS VARIANT) vv FROM variant_rf_a) x " + + "JOIN variant_rf_b b ON x.vv = b.v")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testVariantPushDownContextIsInvalidBeforeTraversal() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + RuntimeFilterContext context = new RuntimeFilterContext( + connectContext.getSessionVariable(), RuntimeFilterId.createGenerator()); + AbstractPhysicalJoin builder = Mockito.mock(AbstractPhysicalJoin.class); + SlotReference scalar = new SlotReference("scalar", IntegerType.INSTANCE); + SlotReference variant = new SlotReference("variant", VariantType.INSTANCE); + Expression cast = new Cast(variant, IntegerType.INSTANCE); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + context.aliasTransferMapPut(scalar, Pair.of(scan, scalar)); + context.aliasTransferMapPut(variant, Pair.of(scan, variant)); + for (TRuntimeFilterType type : ImmutableList.of( + TRuntimeFilterType.IN, TRuntimeFilterType.BLOOM, + TRuntimeFilterType.MIN_MAX, TRuntimeFilterType.IN_OR_BLOOM)) { + Assertions.assertFalse(PushDownContext.createPushDownContextForHashJoin( + variant, scalar, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertFalse(PushDownContext.createPushDownContextForHashJoin( + scalar, variant, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContextForHashJoin( + scalar, cast, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + Assertions.assertTrue(PushDownContext.createPushDownContextForHashJoin( + cast, scalar, context, RuntimeFilterId.createGenerator(), type, builder, false, -1, 0).isValid()); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testScalarCastOfVariantGeneratesRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + for (String sql : ImmutableList.of( + "SELECT * FROM variant_rf_a a JOIN variant_rf_b b " + + "ON CAST(a.v AS INT) = CAST(b.v AS INT)", + "SELECT * FROM (SELECT CAST(v AS INT) vv FROM variant_rf_a) a " + + "JOIN variant_rf_b b ON a.vv = b.k")) { + Assertions.assertFalse(getRuntimeFilters(sql).get().isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + + @Test + public void testVariantSetOperationDoesNotGenerateRuntimeFilter() { + boolean originalEnableVariantV2 = org.apache.doris.common.Config.enable_variant_v2; + org.apache.doris.common.Config.enable_variant_v2 = true; + try { + for (String sql : ImmutableList.of( + "SELECT v FROM variant_rf_a INTERSECT SELECT v FROM variant_rf_b", + "SELECT v FROM variant_rf_a EXCEPT SELECT v FROM variant_rf_b")) { + Assertions.assertTrue(getRuntimeFilters(sql).get().isEmpty(), sql); + Assertions.assertTrue(getSetOperationRuntimeFilters(sql).isEmpty(), sql); + } + } finally { + org.apache.doris.common.Config.enable_variant_v2 = originalEnableVariantV2; + } + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java deleted file mode 100644 index 6598da11c6c36a..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerVariantAutoCastTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.analysis; - -import org.apache.doris.nereids.trees.expressions.functions.scalar.TryParseToVariant; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.types.VariantType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ExpressionAnalyzerVariantAutoCastTest { - - @Test - public void testTargetVariantTypeParticipatesInTryParseEquality() { - StringLiteral json = new StringLiteral("{\"a\":1}"); - VariantType firstTarget = new VariantType(1); - VariantType secondTarget = new VariantType(2); - - TryParseToVariant firstTryParse = new TryParseToVariant(json, firstTarget); - TryParseToVariant secondTryParse = new TryParseToVariant(json, secondTarget); - Assertions.assertNotEquals(firstTryParse, secondTryParse); - Assertions.assertEquals(firstTryParse, new TryParseToVariant(json, new VariantType(1))); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java index e167821dea2e60..d3d2dcea5e5696 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/VariantEqualityContextTest.java @@ -20,9 +20,12 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.thrift.TUniqueId; import org.apache.doris.utframe.TestWithFeService; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; @@ -30,6 +33,8 @@ class VariantEqualityContextTest extends TestWithFeService { + private boolean originalEnableVariantV2; + @Override protected void runBeforeAll() throws Exception { createDatabase("variant_equality_context_test"); @@ -49,6 +54,10 @@ void testLegacyVariantBehaviorIsPreservedWhenV2IsDisabled() { Config.enable_variant_v2 = false; assertRejected("SELECT v, COUNT(*) FROM t1 GROUP BY v", "Doris hll, bitmap"); + assertCheckRejected("SELECT v FROM t1 ORDER BY v", "Doris hll, bitmap"); + assertCheckRejected("SELECT v FROM t1 ORDER BY v LIMIT 10", "Doris hll, bitmap"); + assertRejected("SELECT row_number() OVER (ORDER BY v) FROM t1", "Doris hll, bitmap"); + assertRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", "could not used in ComparisonPredicate"); assertRejected("SELECT DISTINCT v FROM t1", "Doris hll, bitmap"); assertRejected("SELECT COUNT(DISTINCT v) FROM t1", "COUNT DISTINCT"); assertRejected("SELECT v FROM t1 INTERSECT SELECT v FROM t2", "Doris hll, bitmap"); @@ -68,30 +77,71 @@ void testLegacyVariantBehaviorIsPreservedWhenV2IsDisabled() { } } - @Test - void testVariantV2CanonicalHashContextsAndComparisonRestrictions() { - boolean originalEnableVariantV2 = Config.enable_variant_v2; + @BeforeEach + void enableVariantV2() { + originalEnableVariantV2 = Config.enable_variant_v2; Config.enable_variant_v2 = true; - try { - assertAllAccepted( - "SELECT parse_to_variant(CAST(k AS STRING)), COUNT(*) FROM t1 " - + "GROUP BY parse_to_variant(CAST(k AS STRING))", - "SELECT DISTINCT parse_to_variant(CAST(k AS STRING)) FROM t1", - "SELECT COUNT(DISTINCT parse_to_variant(CAST(k AS STRING))) FROM t1", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 INTERSECT " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 EXCEPT " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", - "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 UNION " - + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2"); - - assertVariantComparisonRejected("SELECT parse_to_variant('1') = parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant('1') != parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant('1') <=> parse_to_variant('1.0')"); - assertVariantComparisonRejected("SELECT parse_to_variant(CAST(k AS STRING)) = k FROM t1"); - } finally { - Config.enable_variant_v2 = originalEnableVariantV2; - } + } + + @AfterEach + void restoreVariantV2() { + Config.enable_variant_v2 = originalEnableVariantV2; + } + + @Test + void testRejectedVariantOrderingAndMixedComparisons() { + assertVariantComparisonRejected("SELECT v = k FROM t1"); + assertVariantComparisonRejected("SELECT v > v FROM t1"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); + } + + @Test + void testVariantEquality() { + assertAllAccepted( + "SELECT v = v, v != v, v <=> v FROM t1", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", + "SELECT * FROM t1 LEFT JOIN t2 ON t1.v <=> t2.v", + "SELECT * FROM t1 FULL JOIN t2 ON t1.v = t2.v AND t1.k = t2.k", + "SELECT * FROM t1 JOIN t2 ON t1.v = t2.v OR t1.k = t2.k", + "SELECT EXISTS(SELECT 1 FROM t2 WHERE t1.v = t2.v) FROM t1", + "SELECT * FROM t1 WHERE v IN (SELECT v FROM t2)", + "SELECT * FROM t1 WHERE v NOT IN (SELECT v FROM t2)", + "SELECT * FROM t1 JOIN t2 ON t1.v['id'] = t2.v['id']"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v > t2.v"); + assertVariantComparisonRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.k"); + } + + @Test + void testLegacyVariantEqualityWhenDefaultIsDisabled() { + Config.enable_variant_v2 = false; + assertRejected("SELECT parse_to_variant('1') = parse_to_variant('1.0')", + "could not used in ComparisonPredicate"); + assertRejected("SELECT parse_to_variant('1') <=> NULL", + "could not used in ComparisonPredicate"); + assertRejected("SELECT * FROM t1 JOIN t2 ON t1.v = t2.v", + "could not used in ComparisonPredicate"); + } + + @Test + void testVariantV2CanonicalHashContexts() { + assertAllAccepted( + "SELECT parse_to_variant(CAST(k AS STRING)), COUNT(*) FROM t1 " + + "GROUP BY parse_to_variant(CAST(k AS STRING))", + "SELECT DISTINCT parse_to_variant(CAST(k AS STRING)) FROM t1", + "SELECT COUNT(DISTINCT parse_to_variant(CAST(k AS STRING))) FROM t1", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 INTERSECT " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 EXCEPT " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2", + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t1 UNION " + + "SELECT parse_to_variant(CAST(k AS STRING)) FROM t2"); + } + + @Test + void testVariantOrdering() { + assertPlanAccepted("SELECT v FROM t1 ORDER BY v"); + assertPlanAccepted("SELECT v FROM t1 ORDER BY v LIMIT 10"); + assertPlanAccepted("SELECT row_number() OVER (ORDER BY v) FROM t1"); } @Test @@ -114,6 +164,17 @@ private void assertAllAccepted(String... sqlStatements) { .map(sql -> (Executable) () -> assertAccepted(sql))); } + private void assertPlanAccepted(String sql) { + connectContext.setQueryId(new TUniqueId(1, 1)); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext).plan(sql), sql); + } + + private void assertCheckRejected(String sql, String expectedMessage) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze(sql).applyBottomUp(new CheckAfterRewrite()), sql); + Assertions.assertTrue(exception.getMessage().contains(expectedMessage), exception.getMessage()); + } + private void assertVariantComparisonRejected(String sql) { assertRejected(sql, "CAST to a concrete type first"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index 1f0e6722dc03a0..3fc689f6c289f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -17,10 +17,12 @@ package org.apache.doris.nereids.util; +import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -36,6 +38,8 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArray; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToJson; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; @@ -141,30 +145,29 @@ public void testImplicitCastFromPrimitiveToString() { } @Test - public void testVariantV2ToJsonImplicitCastRequiresExplicitCast() { - boolean originalEnableVariantV2 = Config.enable_variant_v2; - try { - Config.enable_variant_v2 = false; - Assertions.assertEquals(JsonType.INSTANCE, TypeCoercionUtils.implicitCast( - VariantType.INSTANCE, JsonType.INSTANCE).get()); - Config.enable_variant_v2 = true; - Assertions.assertFalse(TypeCoercionUtils.implicitCast( - VariantType.INSTANCE, JsonType.INSTANCE).isPresent()); - } finally { - Config.enable_variant_v2 = originalEnableVariantV2; - } + public void testVariantToJsonImplicitCast() { + Assertions.assertEquals(JsonType.INSTANCE, + TypeCoercionUtils.implicitCast(VariantType.INSTANCE, JsonType.INSTANCE).get()); + } + + @Test + public void testVariantToJsonFunctionSignature() { + Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( + JsonType.INSTANCE, VariantType.INSTANCE)); } @Test - public void testVariantV2ToJsonFunctionSignatureRequiresExplicitCast() { + public void testToJsonAcceptsVariantArgument() { boolean originalEnableVariantV2 = Config.enable_variant_v2; + Config.enable_variant_v2 = true; try { - Config.enable_variant_v2 = false; - Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( - JsonType.INSTANCE, VariantType.INSTANCE)); - Config.enable_variant_v2 = true; - Assertions.assertFalse(ExplicitlyCastableSignature.isExplicitlyCastable( - JsonType.INSTANCE, VariantType.INSTANCE)); + SlotReference variant = new SlotReference("v", new VariantType(100)); + FunctionSignature signature = new ToJson(variant).getSignature(); + Assertions.assertEquals(JsonType.INSTANCE, signature.returnType); + Assertions.assertEquals(variant.getDataType(), signature.getArgType(0)); + // JSON builders keep converting a Variant argument through to_json. + Expression array = new JsonArray(variant).rewriteWhenAnalyze(); + Assertions.assertEquals(new ToJson(variant), array.child(0)); } finally { Config.enable_variant_v2 = originalEnableVariantV2; } @@ -403,22 +406,36 @@ public void testProcessComparisonPredicateDowngrade() { } @Test - public void testVariantV2ComparisonRequiresExplicitCast() { + public void testVariantComparisonRequiresExplicitCast() { boolean originalEnableVariantV2 = Config.enable_variant_v2; Config.enable_variant_v2 = true; - SlotReference variant = new SlotReference("v", VariantType.INSTANCE); - SlotReference anotherVariant = new SlotReference("v2", VariantType.INSTANCE); - SlotReference integer = new SlotReference("i", IntegerType.INSTANCE); - ElementAt variantSubpath = new ElementAt(variant, new StringLiteral("c")); - ElementAt anotherVariantSubpath = new ElementAt(anotherVariant, new StringLiteral("c")); try { - AnalysisException equality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(variant, anotherVariant))); - Assertions.assertTrue(equality.getMessage().contains("CAST to a concrete type first")); - - AnalysisException nullSafeEquality = Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate(new NullSafeEqual(variant, anotherVariant))); - Assertions.assertTrue(nullSafeEquality.getMessage().contains("CAST to a concrete type first")); + SlotReference variant = new SlotReference("v", VariantType.INSTANCE); + SlotReference anotherVariant = new SlotReference("v2", VariantType.INSTANCE); + SlotReference integer = new SlotReference("i", IntegerType.INSTANCE); + ElementAt variantSubpath = new ElementAt(variant, new StringLiteral("c")); + ElementAt anotherVariantSubpath = new ElementAt(anotherVariant, new StringLiteral("c")); + + // Variant equality uses canonical comparison, so it is accepted without a cast. + EqualTo equality = new EqualTo(variant, anotherVariant); + Assertions.assertSame(equality, TypeCoercionUtils.processComparisonPredicate(equality)); + + NullSafeEqual nullSafeEquality = new NullSafeEqual(variant, anotherVariant); + Assertions.assertSame(nullSafeEquality, TypeCoercionUtils.processComparisonPredicate(nullSafeEquality)); + + // A bare NULL takes the Variant type on either side; an ordering comparison stays rejected. + for (ComparisonPredicate bareNull : ImmutableList.of( + new EqualTo(variant, NullLiteral.INSTANCE), new EqualTo(NullLiteral.INSTANCE, variant), + new NullSafeEqual(variant, NullLiteral.INSTANCE), + new NullSafeEqual(NullLiteral.INSTANCE, variant))) { + Expression coerced = TypeCoercionUtils.processComparisonPredicate(bareNull); + Assertions.assertEquals(bareNull.getClass(), coerced.getClass()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(0).getDataType()); + Assertions.assertEquals(VariantType.INSTANCE, coerced.child(1).getDataType()); + } + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate( + new GreaterThan(variant, NullLiteral.INSTANCE))); AnalysisException mixedType = Assertions.assertThrows(AnalysisException.class, () -> TypeCoercionUtils.processComparisonPredicate(new GreaterThan(variant, integer))); @@ -439,18 +456,11 @@ public void testVariantV2ComparisonRequiresExplicitCast() { Assertions.assertEquals(subpathComparison.child(0).getDataType(), subpathComparison.child(1).getDataType()); - Assertions.assertThrows(AnalysisException.class, - () -> TypeCoercionUtils.processComparisonPredicate( - new EqualTo(variantSubpath, anotherVariantSubpath))); + EqualTo subpathEquality = new EqualTo(variantSubpath, anotherVariantSubpath); + Assertions.assertSame(subpathEquality, TypeCoercionUtils.processComparisonPredicate(subpathEquality)); Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( new GreaterThan(new Cast(variant, IntegerType.INSTANCE), integer))); - - Config.enable_variant_v2 = false; - Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( - new EqualTo(variant, integer))); - Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processComparisonPredicate( - new GreaterThan(variant, integer))); } finally { Config.enable_variant_v2 = originalEnableVariantV2; } diff --git a/regression-test/data/nereids_p0/join/test_join_on.out b/regression-test/data/nereids_p0/join/test_join_on.out index b14a14dcc75a94..0b96665bbd0451 100644 --- a/regression-test/data/nereids_p0/join/test_join_on.out +++ b/regression-test/data/nereids_p0/join/test_join_on.out @@ -3,3 +3,7 @@ 1 [1, 2] \N \N 2 [2, 3] \N \N +-- !variant_join -- +1 3 +2 4 + diff --git a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out index 4afc03b5e2b6b0..34632b5b22de47 100644 --- a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out +++ b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out @@ -6,11 +6,11 @@ 4 -9223372036854775808 [1,2,3] {"k":1} 6 1 [1,2,3] {} 7 0 [1,2,3] [1,2,3] -9 \N {} [] +9 \N {} {} 10 1 {} {"n":7,"word":"hot"} 12 1 {"word":"hot","n":7} \N 13 5777142737451291289 {} {"n":7,"word":"hot"} -15 -1 [1,2,3] [] +15 -1 [1,2,3] {} 16 -906638365906932436 {"word":"hot","n":7} {"k":1} 18 -9223372036854775808 {"word":"hot","n":7} {"n":7,"word":"hot"} 19 -9223372036854775808 {"word":"hot","n":7} [1,2,3] diff --git a/regression-test/data/variant_p0/test_variant_bool_path.out b/regression-test/data/variant_p0/test_variant_bool_path.out new file mode 100644 index 00000000000000..affc49e1129fe7 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_bool_path.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_values -- +1 true +2 false +3 true +4 false + +-- !sql_group -- +2 1 +2 2 + +-- !sql_equality -- +1 +3 + +-- !sql_variant_group -- +2 1 +2 2 +1 5 + +-- !sql_rowsets_values_before_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_before_compaction -- +2 1 +2 2 + +-- !sql_rowsets_values_after_compaction -- +1 true +2 false +3 true +4 false + +-- !sql_rowsets_group_after_compaction -- +2 1 +2 2 + +-- !sql_buckets_values -- +1 true +2 false +3 true +4 false +5 true +6 false +7 true +8 false + +-- !sql_buckets_group -- +4 1 +4 2 + diff --git a/regression-test/data/variant_p0/test_variant_join.out b/regression-test/data/variant_p0/test_variant_join.out new file mode 100644 index 00000000000000..2ce98242b952e0 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join.out @@ -0,0 +1,160 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !shuffle -- +1 1 +2 1 +3 2 +4 3 +6 5 +8 7 +9 8 + +-- !null_safe -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + +-- !composite_null_safe -- +1 1 + +-- !left -- +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !full -- +\N 4 +\N 6 +\N 9 +1 1 +10 \N +2 1 +3 2 +4 3 +5 \N +6 5 +7 \N +8 7 +9 8 + +-- !semi -- +1 +2 +3 +4 +6 +8 +9 + +-- !anti -- +10 +5 +7 + +-- !in -- +1 +2 +3 +4 +6 +8 +9 + +-- !not_in -- + +-- !mark -- +1 true +10 \N +2 true +3 true +4 true +5 \N +6 true +7 \N +8 true +9 true + +-- !exists -- +1 true +10 false +2 true +3 true +4 true +5 false +6 true +7 false +8 true +9 true + +-- !subpath -- +9 8 + +-- !residual -- +1 1 +2 1 +2 2 +3 2 +3 3 +4 3 +4 4 +5 5 +6 5 +6 6 +7 7 +8 7 +8 8 +9 8 +9 9 + +-- !scalar_equality -- +1 true false false +10 false true false +2 true false false +3 false true false +4 false true false +5 \N \N true +6 false true false +7 false true false +8 false true false +9 false true false + +-- !json_null_equality -- +true false + +-- !not_in_non_null -- +10 +7 + +-- !typed_key -- +1 1 + +-- !spill -- +1 1 +2 1 +3 2 +4 3 +5 4 +6 5 +8 7 +9 8 + diff --git a/regression-test/data/variant_p0/test_variant_join_type_properties.out b/regression-test/data/variant_p0/test_variant_join_type_properties.out new file mode 100644 index 00000000000000..7ce010d8b2605d --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_join_type_properties.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !broadcast -- +1 1 +2 2 +4 4 + +-- !shuffle -- +1 1 +2 2 +4 4 + +-- !null_safe -- +1 1 +2 2 +3 3 +4 4 + +-- !subpath -- +4 4 + +-- !left_semi -- +1 +2 +4 + +-- !left_anti -- +3 +5 + +-- !mark -- +1 true +2 true +3 \N +4 true +5 \N + +-- !not_in -- +5 + +-- !doc_mode -- +1 1 +4 4 + +-- !computed_key -- +1 1 +1 5 +2 2 +4 4 + +-- !inferred_key -- +1 1 +2 2 + diff --git a/regression-test/data/variant_p0/test_variant_json_functions.out b/regression-test/data/variant_p0/test_variant_json_functions.out new file mode 100644 index 00000000000000..bbf43f503d8849 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_json_functions.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_set_replace_remove -- +1 {"a":1,"b":{"c":[1,2]},"s":"x","z":9} {"a":0,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"r"} {"b":{"c":[1,2]},"s":"x"} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz","z":9} {"a":0,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"r"} {"b":{"c":[3]},"n":{"m":true},"s":"yz"} +6 \N \N \N \N + +-- !extract -- +1 [1,2] x 1 1 1 \N false \N x 1 +2 [3] yz 2 2 2 true false \N yz 2 +3 \N \N \N \N \N \N \N "two" \N \N +4 \N \N \N \N \N \N \N \N \N \N +5 \N \N \N \N \N \N \N \N \N \N +6 \N \N \N \N \N \N \N \N \N \N + +-- !inspect -- +1 ["a", "b", "s"] 3 object true true "$.s" 2600325834747410660 {"a":1,"b.c":[1,2],"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} {"a":1,"b":{"c":[1,2]},"s":"x"} +2 ["a", "b", "n", "s"] 4 object true false \N 8328850723868929851 {"a":2,"b.c":[3],"n.m":true,"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} +3 \N 3 array false \N \N 9010800628419254104 [1,"two",{"three":3}] [1,"two",{"three":3}] [1,"two",{"three":3}] +4 \N 1 int false \N \N 8321545726482568344 42 42 42 +5 \N 1 string false \N \N -1017189637099138661 "abc" "abc" "abc" +6 \N \N \N \N \N \N \N \N \N \N + +-- !construct -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} [{"a":1,"b":{"c":[1,2]},"s":"x"},1] [{"a":1,"b":{"c":[1,2]},"s":"x"}] {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"v":{"a":1,"b":{"c":[1,2]},"s":"x"}} {"b":{"c":[1,2]}} {"v":1} +2 {"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"} [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"},1] [{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}] {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"v":{"a":2,"b":{"c":[3]},"n":{"m":true},"s":"yz"}} {"b":{"c":[3]}} {"v":2} +3 [1,"two",{"three":3}] [[1,"two",{"three":3}],1] [[1,"two",{"three":3}]] {"v":[1,"two",{"three":3}]} {"v":[1,"two",{"three":3}]} {"b":null} {"v":null} +4 42 [42,1] [42] {"v":42} {"v":42} {"b":null} {"v":null} +5 "abc" ["abc",1] ["abc"] {"v":"abc"} {"v":"abc"} {"b":null} {"v":null} +6 \N [null,1] [] {"v":null} {"v":null} {"b":null} {"v":null} + +-- !array_of_variant -- +3 [1,"two",{"three":3}] [1,"two",{"three":3}] [[1,"two",{"three":3}]] + +-- !same_as_explicit_cast -- +1 true true true true true true +2 true true true true true true +3 true true true true true true +4 true true true true true true +5 true true true true true true +6 true true true true true true + +-- !string_signature -- +1 1 "{\\"a\\":1,\\"b\\":{\\"c\\":[1,2]},\\"s\\":\\"x\\"}" {"a":1,"b":{"c":[1,2]},"s":"x"} +4 1 "42" 42 +5 0 "abc" abc + +-- !json_parse -- +1 {"a":1,"b":{"c":[1,2]},"s":"x"} +4 42 + +-- !typed_scalars -- +string "2026-01-02 03:04:05.000000" Decimal32 ["2026-01-02",1.50] + diff --git a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out index ff0c49b7bc445c..ceb60826c894fb 100644 --- a/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out +++ b/regression-test/data/variant_p0/test_variant_ordering_comparison_error.out @@ -1,4 +1,106 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_order -- +2 +3 +1 + +-- !variant_topn -- +1 +3 + +-- !variant_window -- +1 3 +2 1 +3 2 + +-- !asc_nulls_last -- +10 +2 +3 +1 +4 +6 +7 +8 +9 +5 + +-- !desc_nulls_first -- +5 +9 +7 +8 +6 +4 +1 +2 +3 +10 + +-- !rank_peers -- +1 5 4 +2 3 3 +3 3 3 +4 6 5 +5 1 1 +6 7 6 +7 8 7 +8 8 7 +9 10 8 +10 2 2 + +-- !partition_topn -- +5 1 +10 2 +2 3 +3 4 + +-- !spill_sort -- +5 +10 +2 +3 +1 +4 +6 +7 +8 +9 + +-- !spill_sort_desc -- +9 +7 +8 +6 +4 +1 +2 +3 +10 +5 + +-- !bare_null_equals -- +2 \N \N \N \N +5 \N \N \N \N +6 \N \N \N \N + +-- !bare_null_safe_equals -- +5 + +-- !bare_null_safe_equals_reversed -- +5 + +-- !bare_null_safe_not_equals -- +1 +10 +2 +3 +4 +6 +7 +8 +9 + -- !explicit_cast_order -- 1 2 diff --git a/regression-test/data/variant_p0/test_variant_relational_corners.out b/regression-test/data/variant_p0/test_variant_relational_corners.out new file mode 100644 index 00000000000000..54b9646c89aded --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_relational_corners.out @@ -0,0 +1,342 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !constant_types -- +false false false false false + +-- !decimal_types -- +true false true true + +-- !extracted -- +1 1 false +2 1 false +3 1 false +5 0 false +6 -0 false +8 \N true +9 \N true +10 \N true +11 [1,2] false +12 [1,2] false +13 [2,1] false +14 {"a":1,"b":2} false +15 {"a":1,"b":2} false +16 [] false +17 \N true +18 9007199254740992 false +19 9007199254740993 false +20 9007199254740992 false +21 1.5 false +22 false + +-- !groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !distinct -- +11 + +-- !native_order -- +8 +9 +10 +17 +5 +6 +1 +2 +21 +18 +20 +19 +22 +3 +14 +15 +16 +11 +12 +13 + +-- !native_order_desc -- +13 +11 +12 +16 +14 +15 +3 +22 +19 +18 + +-- !roundtrip_types -- +1 false true false +2 false true false +3 false false false +8 \N \N \N +9 \N \N \N +17 \N \N \N + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !sql -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + +-- !cast_precision -- +18 18 +18 19 +18 20 +19 18 +19 19 +19 20 +20 18 +20 19 +20 20 + +-- !cast_groups -- +1 3 + +-- !cast_order -- +5 0 +6 -0 +1 1 +2 1 +21 1.5 +18 9007199254740992 +19 9007199254740992 +20 9007199254740992 + +-- !numeric_promotion -- +1 9007199254740992 +2 9007199254740993 +3 9007199254740992 +4 1 +5 true +6 false +7 0 + +-- !numeric_groups -- +1 2 +2 1 +4 1 +5 1 +6 1 +7 1 + +-- !numeric_join -- +1 1 +1 3 +2 2 +3 1 +3 3 +4 4 +5 5 +6 6 +7 7 + +-- !spill_groups -- +1 2 +3 1 +5 2 +8 4 +11 2 +13 1 +14 2 +16 1 +18 2 +19 1 +21 1 +22 1 + +-- !spill_join -- +1 1 +1 2 +2 1 +2 2 +3 3 +5 5 +5 6 +6 5 +6 6 +8 8 +8 9 +8 10 +8 17 +9 8 +9 9 +9 10 +9 17 +10 8 +10 9 +10 10 +10 17 +11 11 +11 12 +12 11 +12 12 +13 13 +14 14 +14 15 +15 14 +15 15 +16 16 +17 8 +17 9 +17 10 +17 17 +18 18 +18 20 +19 19 +20 18 +20 20 +21 21 +22 22 + diff --git a/regression-test/data/variant_p0/test_variant_subfield_comparison.out b/regression-test/data/variant_p0/test_variant_subfield_comparison.out new file mode 100644 index 00000000000000..681672733562bc --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_subfield_comparison.out @@ -0,0 +1,610 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 \N \N \N \N true \N \N true true true +21 \N 1 \N \N false \N \N false true false +22 1 \N \N \N false \N \N false false true +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 \N \N \N \N true \N \N true true true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 \N \N \N \N true \N \N true true true +37 \N [] \N \N false \N \N false true false +38 \N \N \N \N true \N \N true true true +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +28 +29 +31 +33 +35 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +26 +27 +28 +29 +31 +33 +35 +36 +38 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +30 +32 +34 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +21 +24 +25 +26 +27 +28 +29 +31 +33 +35 +36 +37 +38 + +-- !sql -- +1 1 1 true false true true false true false false +2 1 2 false true false false true false false false +3 -1 -1 true false true true false true false false +4 127 128 false true false false true false false false +5 1 1 true false true true false true false false +6 1 1.5 false true false false true false false false +7 1.5 1.5 true false true true false true false false +8 0 -0 true false true true false true false false +9 9007199254740993 9007199254740992 false true false false true false false false +10 9007199254740992 9007199254740992 true false true true false true false false +11 1 1 false true false false true false false false +12 1.5 1.5 false true false false true false false false +13 abc abc true false true true false true false false +14 abc abd false true false false true false false false +15 true false true true false true false false +16 true true true false true true false true false false +17 true false false true false false true false false false +18 true 1 false true false false true false false false +19 false 0 false true false false true false false false +20 null null true false true true false true false false +21 null 1 false true false false true false false false +22 1 null false true false false true false false false +23 1 \N \N \N false \N \N false false true +24 \N 1 \N \N false \N \N false true false +25 \N \N \N \N true \N \N true true true +26 null \N \N \N false \N \N false false true +27 \N \N \N \N true \N \N true true true +28 [1,2] [1,2] true false true true false true false false +29 [1,2] [1,2] true false true true false true false false +30 [1,2] [2,1] false true false false true false false false +31 [] [] true false true true false true false false +32 [1] 1 false true false false true false false false +33 [1,null,"x",[2]] [1,null,"x",[2]] true false true true false true false false +34 [null] [] false true false false true false false false +35 {"x":1,"y":2} {"x":1,"y":2} true false true true false true false false +36 {} {} true false true true false true false false +37 {} [] false true false false true false false false +38 {"x":null} {} false true false false true false false false +39 [1] ["1"] false true false false true false false false +40 [{"x":1}] [{"x":2}] false true false false true false false false + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !sql -- +2 +4 +6 +9 +11 +12 +14 +17 +18 +19 +21 +22 +30 +32 +34 +37 +38 +39 +40 + +-- !sql -- +1 +3 +5 +7 +8 +10 +13 +15 +16 +20 +24 +25 +27 +28 +29 +31 +33 +35 +36 + +-- !null_object_roundtrip -- +20 true false true false true true +21 true false false false false false +26 true false true true true false +36 true false true false true true +37 true false false false false false +38 true false true false true false + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false false true false false true false +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- + +-- !sql -- +1 +2 + +-- !sql -- +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +1 +3 +2 +4 +5 + +-- !sql -- +4 +5 +2 +1 +3 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !sql -- +1 false false true false true true false true +2 false false false true false false true false +3 true false \N \N false \N \N false +4 false true \N \N false \N \N false +5 true true \N \N true \N \N true + +-- !sql -- +1 + +-- !sql -- +2 + +-- !sql -- +1 +5 + +-- !sql -- +2 +1 +3 +4 +5 + +-- !sql -- +4 +5 +1 +3 +2 + +-- !explicit_numeric_cast -- +1 true false false +2 false false true +3 true false false +4 false false true +5 true false false +6 false false true +7 true false false +8 true false false +9 true false false +10 true false false +11 true false false +12 true false false + +-- !typed_projection -- +1 1 1 true false true +2 1 1.5 false true false +3 0 -0 true false true +4 9007199254740993 9007199254740992 false true false +5 9007199254740992 9007199254740992 true false true +6 \N 1 \N \N false +7 1 \N \N \N false +8 \N \N \N \N true +9 \N \N \N \N true + +-- !sql -- +1 +3 +5 + +-- !sql -- +2 +4 + +-- !sql -- +1 +3 +5 +8 +9 + diff --git a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out b/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out deleted file mode 100644 index 05485b53ca8152..00000000000000 --- a/regression-test/data/variant_p0/test_variant_timestamptz_sparse.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_all -- -1 2003-12-21 10:29:00.000000+08:00 2003-12-21 10:29:00.000000+08:00 2003-12-21 10:29:00.000000+08:00 2003-12-21 15:29:00.000000+08:00 2003-12-21 15:29:00.000000+08:00 2003-12-21 07:29:00.000000+08:00 - --- !select_cast_ts0 -- -1 2003-12-21 10:29:00+08:00 2003-12-21 10:29:00.000+08:00 2003-12-21 15:29:00.000000+08:00 - diff --git a/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out b/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out new file mode 100644 index 00000000000000..2d7ef477cf5f15 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_v1_relational_compatibility.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !scalar_comparison -- +true true true + +-- !json_extract -- +1 + diff --git a/regression-test/data/variant_p0/test_variant_window_partition.out b/regression-test/data/variant_p0/test_variant_window_partition.out new file mode 100644 index 00000000000000..c2c30815eb02b1 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_window_partition.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_by_subpath -- +1 1 1 +2 1 2 +3 2 1 +4 a 1 +5 true 1 +6 true 2 +7 1 3 +8 true 3 + +-- !partition_by_whole -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_rank -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 +7 3 +8 3 + +-- !partition_topn -- +1 1 +2 2 +3 1 +4 1 +5 1 +6 2 + +-- !partition_mixed_keys -- +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 2 +8 2 + +-- !many_blocks_typed_keys -- +5000 12502500 4169167 + +-- !many_blocks_encoded_keys -- +5000 12502500 4169167 + diff --git a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out index 80fea8c68d269b..9e635b956753a9 100644 --- a/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out +++ b/regression-test/data/variant_p0/v2/test_variant_equality_contexts.out @@ -1,4 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !equals -- +true + +-- !not_equals -- +false + +-- !null_safe_equals -- +true + +-- !equality_join -- +0 0 +1 1 + +-- !null_safe_equality_join -- +0 0 +1 1 + -- !group_by -- 0 2 1 2 @@ -36,3 +53,4 @@ false null 1 -- !explicit_cast_equality -- true true + diff --git a/regression-test/suites/nereids_p0/join/test_join_on.groovy b/regression-test/suites/nereids_p0/join/test_join_on.groovy index 7de75912796cc6..d39105c7a60e06 100644 --- a/regression-test/suites/nereids_p0/join/test_join_on.groovy +++ b/regression-test/suites/nereids_p0/join/test_join_on.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_join_on", "nereids_p0") { +suite("test_join_on", "nereids_p0,nonConcurrent") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" @@ -47,8 +47,12 @@ suite("test_join_on", "nereids_p0") { exception "errCode = 2" } - test { - sql """select * from (select cast('' as variant) as a) t1 join (select cast('' as variant) as a) t2 on t1.a = t2.a""" - exception "could not used in ComparisonPredicate (a = a)" + setFeConfigTemporary([enable_variant_v2: true]) { + order_qt_variant_join """ + select t1.id, t2.id + from (select 1 as id, cast('x' as variant) as a union all select 2, cast(1 as variant)) t1 + join (select 3 as id, parse_to_variant('"x"') as a union all select 4, parse_to_variant('1.0')) t2 + on t1.a = t2.a + """ } } diff --git a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy index c2c6c236f1d68b..8e7023ccfafb8c 100644 --- a/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy +++ b/regression-test/suites/variant_p0/predefine/predefined_typed_to_sparse_1shard.groovy @@ -41,7 +41,6 @@ suite("test_predefine_typed_to_sparse_1shard", "p0,nonConcurrent") { cast(var['char_1'] as text)""" sql """ set enable_common_expr_pushdown = true """ sql """ set default_variant_enable_doc_mode = false """ - // sql """ set enable_variant_schema_auto_cast = false """ def load_json_data = {table_name, file_name -> // load the json data diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy index f832ccc716d1f1..89da23cdf36951 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy @@ -107,6 +107,12 @@ suite("test_predefine_ddl", "p0") { sql """ alter table ${tableName} add column var3 variant<'ab' : string> NULL """ + // A Schema Template cannot declare TIMESTAMPTZ paths. + test { + sql """ alter table ${tableName} add column var4 variant<'ts' : timestamptz(3)> NULL """ + exception("VARIANT unsupported sub-type: timestamptz(3)") + } + // TODO(lihangyu) : uncomment // test { // sql """ alter table ${tableName} modify column var variant<'ab' : string> NULL """ @@ -202,6 +208,30 @@ suite("test_predefine_ddl", "p0") { exception("VARIANT unsupported sub-type: datetime") } + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : timestamptz(6) + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: timestamptz(6)") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + 'ab*' : array + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: array r.v""" + order_qt_composite_null_safe """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v <=> r.v AND l.id % 2 = r.id % 2""" + order_qt_left """SELECT l.id, r.id FROM variant_join_left l + LEFT JOIN variant_join_right r ON l.v = r.v""" + order_qt_full """SELECT l.id, r.id FROM variant_join_left l + FULL JOIN variant_join_right r ON l.v = r.v""" + order_qt_semi """SELECT l.id FROM variant_join_left l + LEFT SEMI JOIN variant_join_right r ON l.v = r.v""" + order_qt_anti """SELECT l.id FROM variant_join_left l + LEFT ANTI JOIN variant_join_right r ON l.v = r.v""" + order_qt_in """SELECT id FROM variant_join_left + WHERE v IN (SELECT v FROM variant_join_right)""" + order_qt_not_in """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right)""" + order_qt_mark """SELECT l.id, l.v IN (SELECT r.v FROM variant_join_right r) + FROM variant_join_left l""" + order_qt_exists """SELECT l.id, EXISTS(SELECT 1 FROM variant_join_right r WHERE l.v = r.v) + FROM variant_join_left l""" + order_qt_subpath """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v['id'] = r.v['id']""" + order_qt_residual """SELECT l.id, r.id FROM variant_join_left l + JOIN variant_join_right r ON l.v = r.v OR l.id = r.id""" + order_qt_scalar_equality """SELECT l.id, l.v = parse_to_variant('1.0'), + l.v != parse_to_variant('1.0'), l.v <=> CAST(NULL AS VARIANT) + FROM variant_join_left l""" + qt_json_null_equality """SELECT parse_to_variant('null') = parse_to_variant('null'), + parse_to_variant('null') = parse_to_variant('{}')""" + // Without a NULL on the build side, NOT IN depends on canonical key matching. + order_qt_not_in_non_null """SELECT id FROM variant_join_left + WHERE v NOT IN (SELECT v FROM variant_join_right WHERE v IS NOT NULL)""" + // A typed key from CAST(scalar AS VARIANT) against a stored Variant key. + order_qt_typed_key """SELECT l.id, r.id + FROM (SELECT id, CAST(CAST(id AS BIGINT) AS VARIANT) k FROM variant_join_left) l + JOIN variant_join_right r ON l.k = r.v""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + order_qt_spill """SELECT l.id, r.id FROM variant_join_left l + JOIN [shuffle] variant_join_right r ON l.v <=> r.v""" + test { + sql "SELECT * FROM variant_join_left l JOIN variant_join_right r ON l.v > r.v" + exception "CAST to a concrete type first" + } + } +} diff --git a/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy new file mode 100644 index 00000000000000..d6088108bccb93 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_join_type_properties.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_join_type_properties", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + // Variant join keys compare canonically. Keys whose VARIANT types differ only in + // properties (max subcolumns count, doc mode) must join like identical types. + sql "DROP TABLE IF EXISTS variant_join_props_a" + sql "DROP TABLE IF EXISTS variant_join_props_b" + sql "DROP TABLE IF EXISTS variant_join_props_c" + sql "DROP TABLE IF EXISTS variant_join_props_d" + sql """CREATE TABLE variant_join_props_a ( + id INT, v VARIANT, s STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_b ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_c ( + id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1")""" + sql """CREATE TABLE variant_join_props_d ( + id INT, + v VARIANT, + w VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_join_props_a VALUES + (1, parse_to_variant('1'), '1.0'), (2, parse_to_variant('"x"'), '"x"'), + (3, NULL, NULL), (4, parse_to_variant('{"k":1,"j":[1,2]}'), '{"j":[1.0,2],"k":1.0}'), + (5, parse_to_variant('true'), '1')""" + sql """INSERT INTO variant_join_props_b VALUES + (1, parse_to_variant('1.0')), (2, parse_to_variant('"x"')), (3, NULL), + (4, parse_to_variant('{"j":[1,2.0],"k":1}')), (6, parse_to_variant('false'))""" + sql """INSERT INTO variant_join_props_c VALUES + (1, parse_to_variant('1')), (4, parse_to_variant('{"k":1.0,"j":[1,2]}'))""" + sql """INSERT INTO variant_join_props_d VALUES + (1, parse_to_variant('1'), parse_to_variant('1.0')), + (2, parse_to_variant('"x"'), parse_to_variant('"x"')), + (4, parse_to_variant('{"k":1}'), parse_to_variant('{"k":2}'))""" + + // Different max subcolumns counts. + order_qt_broadcast """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [broadcast] variant_join_props_b b ON a.v = b.v""" + order_qt_shuffle """SELECT a.id, b.id FROM variant_join_props_a a + JOIN [shuffle] variant_join_props_b b ON a.v = b.v""" + order_qt_null_safe """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v <=> b.v""" + order_qt_subpath """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.v['j'] = b.v['j']""" + order_qt_left_semi """SELECT a.id FROM variant_join_props_a a + LEFT SEMI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_left_anti """SELECT a.id FROM variant_join_props_a a + LEFT ANTI JOIN variant_join_props_b b ON a.v = b.v""" + order_qt_mark """SELECT a.id, a.v IN (SELECT b.v FROM variant_join_props_b b) + FROM variant_join_props_a a""" + order_qt_not_in """SELECT a.id FROM variant_join_props_a a + WHERE a.v NOT IN (SELECT b.v FROM variant_join_props_b b WHERE b.v IS NOT NULL)""" + // Different doc mode. + order_qt_doc_mode """SELECT a.id, c.id FROM variant_join_props_a a + JOIN variant_join_props_c c ON a.v = c.v""" + // A stored column against a computed Variant whose type has default properties. + order_qt_computed_key """SELECT l.id, r.id FROM variant_join_props_a l + JOIN variant_join_props_a r ON l.v = parse_to_variant(r.s)""" + // The optimizer infers a.v = b.v from d.v = a.v, d.w = b.v and d.v = d.w and uses it + // as a hash key between tables whose Variant properties differ. + sql "SET disable_join_reorder = true" + order_qt_inferred_key """SELECT a.id, b.id FROM variant_join_props_a a + JOIN variant_join_props_b b ON a.id = b.id + JOIN variant_join_props_d d ON d.v = a.v AND d.w = b.v + WHERE d.v = d.w""" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_json_functions.groovy b/regression-test/suites/variant_p0/test_variant_json_functions.groovy new file mode 100644 index 00000000000000..52e62690746501 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// JSON functions accept a Variant argument by casting it to JSON implicitly. +suite("test_variant_json_functions", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + sql "DROP TABLE IF EXISTS test_variant_json_functions" + sql """ + CREATE TABLE test_variant_json_functions (k INT, v VARIANT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO test_variant_json_functions + SELECT k, parse_to_variant(doc) FROM ( + SELECT 1 AS k, '{"a":1,"s":"x","b":{"c":[1,2]}}' AS doc + UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}' + UNION ALL SELECT 3, '[1,"two",{"three":3}]' + UNION ALL SELECT 4, '42' + UNION ALL SELECT 5, 'abc' + UNION ALL SELECT 6, NULL) t + """ + + // json_insert and json_set reject a scalar document, so they only run on object rows. + order_qt_insert_set_replace_remove """ + SELECT k, json_insert(v, '\$.z', 9), json_set(v, '\$.a', 0), json_replace(v, '\$.s', 'r'), + json_remove(v, '\$.a') + FROM test_variant_json_functions WHERE k IN (1, 2, 6) ORDER BY k + """ + order_qt_extract """ + SELECT k, json_extract(v, '\$.b.c'), json_extract_string(v, '\$.s'), json_extract_int(v, '\$.a'), + json_extract_bigint(v, '\$.a'), json_extract_double(v, '\$.a'), json_extract_bool(v, '\$.n.m'), + json_extract_isnull(v, '\$.a'), jsonb_extract(v, '\$[1]'), get_json_string(v, '\$.s'), + get_json_int(v, '\$.a') + FROM test_variant_json_functions ORDER BY k + """ + order_qt_inspect """ + SELECT k, json_keys(v), json_length(v), json_type(v, '\$'), json_exists_path(v, '\$.s'), + json_contains(v, '1', '\$.a'), json_search(v, 'one', 'x'), json_hash(v), + json_object_flatten(v), sort_json_object_keys(v), normalize_json_numbers_to_double(v) + FROM test_variant_json_functions ORDER BY k + """ + // Variant values nested into new JSON documents keep their JSON structure. + order_qt_construct """ + SELECT k, to_json(v), json_array(v, 1), json_array_ignore_null(v, NULL), json_object('v', v), + json_insert('{}', '\$.v', v), json_set('{}', '\$.b', v['b']), json_replace('{"v":0}', '\$.v', v['a']) + FROM test_variant_json_functions ORDER BY k + """ + + // Variant elements of an ARRAY convert the same way. + order_qt_array_of_variant """ + SELECT k, to_json(CAST(v AS ARRAY)), CAST(CAST(v AS ARRAY) AS JSON), + json_array(CAST(v AS ARRAY)) + FROM test_variant_json_functions WHERE k = 3 + """ + + // The implicit cast gives the same result as the explicit one. + order_qt_same_as_explicit_cast """ + SELECT k, + CAST(json_replace(v, '\$.s', 'r') AS STRING) <=> CAST(json_replace(CAST(v AS JSON), '\$.s', 'r') AS STRING), + CAST(json_extract(v, '\$.b') AS STRING) <=> CAST(json_extract(CAST(v AS JSON), '\$.b') AS STRING), + CAST(json_keys(v) AS STRING) <=> CAST(json_keys(CAST(v AS JSON)) AS STRING), + json_hash(v) <=> json_hash(CAST(v AS JSON)), + CAST(to_json(v) AS STRING) <=> CAST(CAST(v AS JSON) AS STRING), + CAST(json_array(v) AS STRING) <=> CAST(json_array(CAST(v AS JSON)) AS STRING) + FROM test_variant_json_functions ORDER BY k + """ + + // Functions with a string signature keep treating Variant as text, so an unparsable document + // that was stored as a Variant string is still reported as invalid JSON. + order_qt_string_signature """ + SELECT k, json_valid(v), json_quote(v), json_unquote(v) + FROM test_variant_json_functions WHERE k IN (1, 4, 5) ORDER BY k + """ + order_qt_json_parse """ + SELECT k, json_parse(v) FROM test_variant_json_functions WHERE k IN (1, 4) ORDER BY k + """ + + // Typed Variant scalars without a JSON counterpart become JSON strings. + order_qt_typed_scalars """ + SELECT json_type(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), '\$'), + json_extract(CAST(CAST('2026-01-02 03:04:05' AS DATETIME) AS VARIANT), '\$'), + json_type(CAST(CAST(1.50 AS DECIMAL(5, 2)) AS VARIANT), '\$'), + json_array(CAST(CAST('2026-01-02' AS DATE) AS VARIANT), CAST(1.50 AS DECIMAL(5, 2))) + """ + } +} diff --git a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy index ee0b25cb62f9db..1e329c1c10588c 100644 --- a/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy +++ b/regression-test/suites/variant_p0/test_variant_ordering_comparison_error.groovy @@ -16,54 +16,91 @@ // under the License. suite("test_variant_ordering_comparison_error", "p0,nonConcurrent") { - def enableVariantV2 = getFeConfig("enable_variant_v2").toBoolean() + setFeConfigTemporary([enable_variant_v2: true]) { + sql "SET enable_nereids_planner = true" sql "SET enable_fallback_to_original_planner = false" - test { - sql """ - SELECT v + qt_variant_order """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v + ORDER BY v, id """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" - } - - test { - sql """ - SELECT v + qt_variant_topn """ + SELECT id FROM ( - SELECT CAST('2' AS VARIANT) AS v + SELECT 1 AS id, parse_to_variant('2') AS v UNION ALL - SELECT CAST('1' AS VARIANT) AS v + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v ) t - ORDER BY v - LIMIT 1 + ORDER BY v DESC, id + LIMIT 2 """ - exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" + qt_variant_window """ + SELECT id, row_number() OVER (ORDER BY v, id) + FROM ( + SELECT 1 AS id, parse_to_variant('2') AS v + UNION ALL + SELECT 2 AS id, parse_to_variant('1') AS v + UNION ALL + SELECT 3 AS id, parse_to_variant('1.5') AS v + ) t + ORDER BY id + """ + // Canonical peers (1, 1.0 and objects with reordered keys), SQL NULL, a JSON null (read back + // as {}) and a boolean. One bucket keeps the numbers ahead of the boolean in the segment. + sql "DROP TABLE IF EXISTS variant_ordering_peers" + sql """CREATE TABLE variant_ordering_peers (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1")""" + sql """INSERT INTO variant_ordering_peers VALUES + (1, parse_to_variant('2')), (2, parse_to_variant('1')), (3, parse_to_variant('1.0')), + (4, parse_to_variant('"a"')), (5, NULL), (6, parse_to_variant('null')), + (7, parse_to_variant('{"k":1,"j":2}')), (8, parse_to_variant('{"j":2,"k":1.0}')), + (9, parse_to_variant('[1,2]')), (10, parse_to_variant('true'))""" + qt_asc_nulls_last "SELECT id FROM variant_ordering_peers ORDER BY v ASC NULLS LAST, id" + qt_desc_nulls_first "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS FIRST, id" + qt_rank_peers """SELECT id, rank() OVER (ORDER BY v), dense_rank() OVER (ORDER BY v) + FROM variant_ordering_peers ORDER BY id""" + qt_partition_topn """SELECT id, rn FROM ( + SELECT id, row_number() OVER (ORDER BY v, id) rn FROM variant_ordering_peers) t + WHERE rn <= 4 ORDER BY rn""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_sort "SELECT id FROM variant_ordering_peers ORDER BY v NULLS FIRST, id" + qt_spill_sort_desc "SELECT id FROM variant_ordering_peers ORDER BY v DESC NULLS LAST, id" + sql "SET enable_force_spill = false" + sql "SET enable_spill = false" + + // A bare NULL takes the Variant type on either side. `=` and `!=` are NULL for every row; + // `<=>` matches the SQL NULL row only. + order_qt_bare_null_equals """SELECT id, v = NULL, NULL = v, v != NULL, NULL != v + FROM variant_ordering_peers WHERE id IN (2, 5, 6)""" + order_qt_bare_null_safe_equals "SELECT id FROM variant_ordering_peers WHERE v <=> NULL" + order_qt_bare_null_safe_equals_reversed "SELECT id FROM variant_ordering_peers WHERE NULL <=> v" + order_qt_bare_null_safe_not_equals "SELECT id FROM variant_ordering_peers WHERE NOT (v <=> NULL)" + test { + sql "SELECT id FROM variant_ordering_peers WHERE v > NULL" + exception "CAST to a concrete type first" } test { sql "SELECT CAST('2' AS VARIANT) > CAST('1' AS VARIANT)" - exception enableVariantV2 - ? "CAST to a concrete type first" - : "could not used in ComparisonPredicate" + exception "CAST to a concrete type first" } - if (enableVariantV2) { - test { - sql "SELECT CAST('2' AS VARIANT) > 1" - exception "CAST to a concrete type first" - } - - test { - sql "SELECT CAST('2' AS VARIANT) <=> CAST('1' AS VARIANT)" - exception "CAST to a concrete type first" - } + test { + sql "SELECT CAST('2' AS VARIANT) > 1" + exception "CAST to a concrete type first" } order_qt_explicit_cast_order """ @@ -111,4 +148,5 @@ suite("test_variant_ordering_comparison_error", "p0,nonConcurrent") { exception "Doris hll, bitmap, array, map, struct, jsonb, variant column" } + } } diff --git a/regression-test/suites/variant_p0/test_variant_relational_corners.groovy b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy new file mode 100644 index 00000000000000..c255471a2282fa --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_relational_corners.groovy @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_relational_corners", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + qt_constant_types """SELECT + parse_to_variant('true') = parse_to_variant('1'), + parse_to_variant('false') = parse_to_variant('0'), + parse_to_variant('null') = parse_to_variant('{}'), + parse_to_variant('{"k":{}}')['k'] IS NULL, + parse_to_variant('9007199254740993') = parse_to_variant('9007199254740992.0')""" + qt_decimal_types """SELECT + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) + = CAST(CAST(1.500 AS DECIMAL(10,3)) AS VARIANT), + CAST(CAST(1.50 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1.5'), + CAST(CAST(1.00 AS DECIMAL(10,2)) AS VARIANT) = parse_to_variant('1'), + CAST(CAST(-1 AS TINYINT) AS VARIANT) = CAST(CAST(-1 AS BIGINT) AS VARIANT)""" + sql "DROP TABLE IF EXISTS variant_relational_corners" + sql """CREATE TABLE variant_relational_corners (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num"="1")""" + // Keep values inside objects so extraction exercises shredded subcolumns, + // mixed-type promotion and the missing-path validity map. Booleans are compared in + // variant_relational_numeric instead: in this bucket layout a segment can see a boolean + // before a number, and storage then widens the boolean to an integer. + sql """INSERT INTO variant_relational_corners VALUES + (1, parse_to_variant('{"k":1}')), + (2, parse_to_variant('{"k":1.0}')), + (3, parse_to_variant('{"k":"1"}')), + (5, parse_to_variant('{"k":0}')), + (6, parse_to_variant('{"k":-0.0}')), + (8, parse_to_variant('{"k":null}')), + (9, parse_to_variant('{}')), + (10, NULL), + (11, parse_to_variant('{"k":[1,2]}')), + (12, parse_to_variant('{"k":[1.0,2.0]}')), + (13, parse_to_variant('{"k":[2,1]}')), + (14, parse_to_variant('{"k":{"a":1,"b":2}}')), + (15, parse_to_variant('{"k":{"b":2.0,"a":1.0}}')), + (16, parse_to_variant('{"k":[]}')), + (17, parse_to_variant('{"k":{}}')), + (18, parse_to_variant('{"k":9007199254740992}')), + (19, parse_to_variant('{"k":9007199254740993}')), + (20, parse_to_variant('{"k":9007199254740992.0}')), + (21, parse_to_variant('{"k":1.5}')), + (22, parse_to_variant('{"k":""}'))""" + + qt_extracted """SELECT id, v['k'], v['k'] IS NULL + FROM variant_relational_corners ORDER BY id""" + qt_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_distinct "SELECT count(DISTINCT v['k']) FROM variant_relational_corners" + qt_native_order """SELECT id FROM variant_relational_corners + ORDER BY v['k'] NULLS FIRST, id""" + qt_native_order_desc """SELECT id FROM variant_relational_corners + ORDER BY v['k'] DESC NULLS LAST, id LIMIT 10""" + // Compare stored values to constants explicitly: generated output is an + // observation of round-trip behavior, not a definition of type coercion. + qt_roundtrip_types """SELECT id, v['k'] = parse_to_variant('true'), + v['k'] = parse_to_variant('1'), v['k'] = parse_to_variant('{}') + FROM variant_relational_corners WHERE id IN (1,2,3,8,9,17) + ORDER BY id""" + ["broadcast", "shuffle"].each { distribution -> + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] = r.v['k'] + ORDER BY l.id, r.id""" + qt_sql """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [${distribution}] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" + } + // CAST is intentionally a different oracle here: strings can merge with + // numbers and DOUBLE cannot distinguish adjacent integers above 2^53. + qt_cast_precision """SELECT l.id, r.id FROM variant_relational_corners l + JOIN variant_relational_corners r + ON CAST(l.v['k'] AS DOUBLE) = CAST(r.v['k'] AS DOUBLE) + WHERE l.id BETWEEN 18 AND 20 AND r.id BETWEEN 18 AND 20 + ORDER BY l.id, r.id""" + qt_cast_groups """SELECT min(id), count(*) FROM variant_relational_corners + WHERE id IN (1,2,3) GROUP BY CAST(v['k'] AS BIGINT) ORDER BY min(id)""" + qt_cast_order """SELECT id, CAST(v['k'] AS DOUBLE) + FROM variant_relational_corners WHERE id IN (1,2,5,6,18,19,20,21) + ORDER BY CAST(v['k'] AS DOUBLE), id""" + sql "DROP TABLE IF EXISTS variant_relational_numeric" + sql """CREATE TABLE variant_relational_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_numeric VALUES + (1, parse_to_variant('{"k":9007199254740992}')), + (2, parse_to_variant('{"k":9007199254740993}')), + (3, parse_to_variant('{"k":9007199254740992.0}')), + (4, parse_to_variant('{"k":1}')), + (5, parse_to_variant('{"k":true}')), + (6, parse_to_variant('{"k":false}')), + (7, parse_to_variant('{"k":0}'))""" + qt_numeric_promotion """SELECT id, v['k'] FROM variant_relational_numeric ORDER BY id""" + qt_numeric_groups """SELECT min(id), count(*) FROM variant_relational_numeric + GROUP BY v['k'] ORDER BY min(id)""" + qt_numeric_join """SELECT l.id, r.id FROM variant_relational_numeric l + JOIN variant_relational_numeric r ON l.v['k'] = r.v['k'] ORDER BY l.id,r.id""" + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + qt_spill_groups """SELECT min(id), count(*) FROM variant_relational_corners + GROUP BY v['k'] ORDER BY min(id)""" + qt_spill_join """SELECT l.id, r.id FROM variant_relational_corners l + JOIN [shuffle] variant_relational_corners r ON l.v['k'] <=> r.v['k'] + ORDER BY l.id, r.id""" + } +} diff --git a/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy new file mode 100644 index 00000000000000..aa8de479fcaa65 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_subfield_comparison.groovy @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_subfield_comparison", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + // Keep the JSON text as well as its stored Variant representation: reading a + // subcolumn can promote types, while parsing the text preserves each input. + sql "DROP TABLE IF EXISTS variant_subfield_comparison" + sql """CREATE TABLE variant_subfield_comparison (id INT, doc STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_comparison (id, doc) VALUES + (1, '{"a":1,"b":1}'), + (2, '{"a":1,"b":2}'), + (3, '{"a":-1,"b":-1}'), + (4, '{"a":127,"b":128}'), + (5, '{"a":1,"b":1.0}'), + (6, '{"a":1,"b":1.5}'), + (7, '{"a":1.5,"b":1.5}'), + (8, '{"a":0,"b":-0.0}'), + (9, '{"a":9007199254740993,"b":9007199254740992.0}'), + (10, '{"a":9007199254740992,"b":9007199254740992.0}'), + (11, '{"a":"1","b":1}'), + (12, '{"a":"1.5","b":1.5}'), + (13, '{"a":"abc","b":"abc"}'), + (14, '{"a":"abc","b":"abd"}'), + (15, '{"a":"","b":""}'), + (16, '{"a":true,"b":true}'), + (17, '{"a":true,"b":false}'), + (18, '{"a":true,"b":1}'), + (19, '{"a":false,"b":0}'), + (20, '{"a":null,"b":null}'), + (21, '{"a":null,"b":1}'), + (22, '{"a":1,"b":null}'), + (23, '{"a":1}'), + (24, '{"b":1}'), + (25, '{}'), + (26, '{"a":null}'), + (27, NULL), + (28, '{"a":[1,2],"b":[1,2]}'), + (29, '{"a":[1,2],"b":[1.0,2.0]}'), + (30, '{"a":[1,2],"b":[2,1]}'), + (31, '{"a":[],"b":[]}'), + (32, '{"a":[1],"b":1}'), + (33, '{"a":[1,null,"x",[2]],"b":[1.0,null,"x",[2.0]]}'), + (34, '{"a":[null],"b":[]}'), + (35, '{"a":{"x":1,"y":2},"b":{"y":2.0,"x":1.0}}'), + (36, '{"a":{},"b":{}}'), + (37, '{"a":{},"b":[]}'), + (38, '{"a":{"x":null},"b":{}}'), + (39, '{"a":[1],"b":["1"]}'), + (40, '{"a":[{"x":1}],"b":[{"x":2}]}')""" + // Insert into a second table to retain both original text and parsed values. + sql "DROP TABLE IF EXISTS variant_subfield_values" + sql """CREATE TABLE variant_subfield_values (id INT, doc STRING, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_values + SELECT id, doc, parse_to_variant(doc) FROM variant_subfield_comparison""" + + // These are separate round-trip expectations, not an assertion that storage + // preserves JSON null and empty objects. Rows 20-27 and 36-38 expose the + // current difference between stored-path SQL NULL and parsed JSON values. + ["v", "parse_to_variant(doc)"].each { expression -> + qt_sql """SELECT id, x['a'], x['b'], + x['a'] = x['b'], x['a'] != x['b'], x['a'] <=> x['b'], + x['b'] = x['a'], x['b'] != x['a'], x['b'] <=> x['a'], + x['a'] IS NULL, x['b'] IS NULL + FROM (SELECT id, ${expression} x FROM variant_subfield_values) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b'] ORDER BY id""" + } + qt_sql """SELECT id FROM variant_subfield_values + WHERE NOT (${expression}['a'] = ${expression}['b']) ORDER BY id""" + qt_sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] = ${expression}['b'] + OR ${expression}['a'] IS NULL ORDER BY id""" + } + // TopN ordering is supported; relational ordering predicates still require CAST. + ["v", "parse_to_variant(doc)"].each { expression -> + [">", ">=", "<", "<="].each { op -> + test { + sql "SELECT ${expression}['a'] ${op} ${expression}['b'] FROM variant_subfield_values" + exception "CAST to a concrete type first" + } + test { + sql """SELECT id FROM variant_subfield_values + WHERE ${expression}['a'] ${op} ${expression}['b']""" + exception "CAST to a concrete type first" + } + } + } + // Isolate the known storage round-trip differences from ordinary equality cases. + qt_null_object_roundtrip """SELECT id, + v['a'] IS NULL, parse_to_variant(doc)['a'] IS NULL, + v['b'] IS NULL, parse_to_variant(doc)['b'] IS NULL, + v['a'] <=> v['b'], + parse_to_variant(doc)['a'] <=> parse_to_variant(doc)['b'] + FROM variant_subfield_values WHERE id IN (20,21,26,36,37,38) ORDER BY id""" + + // JSON text cannot retain Decimal, date or IP physical types. Cast scalar + // columns to Variant directly to exercise these same-row comparison types. + sql "DROP TABLE IF EXISTS variant_comparison_scalar_sources" + sql """CREATE TABLE variant_comparison_scalar_sources + (id INT, a STRING, b STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + [ + ["DECIMAL(18,2)", "DECIMAL(38,3)", "1.50", "1.500", "1.501"], + ["DECIMAL(18,2)", "DOUBLE", "1.50", "1.5", "1.6"], + ["DECIMAL(18,2)", "BIGINT", "1.00", "1", "2"], + ["DATEV2", "DATEV2", "2024-02-29", "2024-02-29", "2024-03-01"], + ["DATETIMEV2(3)", "DATETIMEV2(6)", "2024-02-29 12:34:56.123", + "2024-02-29 12:34:56.123000", "2024-02-29 12:34:56.123001"], + ["IPV4", "IPV4", "10.0.0.2", "10.0.0.2", "10.0.0.10"], + ["IPV6", "IPV6", "2001:db8::1", "2001:0db8:0:0:0:0:0:1", "2001:db8::2"], + ["DOUBLE", "DOUBLE", "NaN", "NaN", "Infinity"], + ["DOUBLE", "DOUBLE", "Infinity", "Infinity", "-Infinity"] + ].each { spec -> + sql "TRUNCATE TABLE variant_comparison_scalar_sources" + sql """INSERT INTO variant_comparison_scalar_sources VALUES + (1, '${spec[2]}', '${spec[3]}'), (2, '${spec[2]}', '${spec[4]}'), + (3, NULL, '${spec[3]}'), (4, '${spec[2]}', NULL), (5, NULL, NULL)""" + def source = """SELECT id, CAST(CAST(a AS ${spec[0]}) AS VARIANT) a, + CAST(CAST(b AS ${spec[1]}) AS VARIANT) b + FROM variant_comparison_scalar_sources""" + qt_sql """SELECT id, a IS NULL, b IS NULL, + a = b, a != b, a <=> b, b = a, b != a, b <=> a + FROM (${source}) t ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql "SELECT id FROM (${source}) t WHERE a ${op} b ORDER BY id" + } + // Typed Variant values also order natively; cover both NULL placements. + qt_sql "SELECT id FROM (${source}) t ORDER BY b ASC NULLS LAST, id" + qt_sql "SELECT id FROM (${source}) t ORDER BY b DESC NULLS FIRST, id" + } + qt_explicit_numeric_cast """SELECT id, + CAST(v['a'] AS DOUBLE) = CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) > CAST(v['b'] AS DOUBLE), + CAST(v['a'] AS DOUBLE) < CAST(v['b'] AS DOUBLE) + FROM variant_subfield_values WHERE id BETWEEN 1 AND 12 ORDER BY id""" + + // Homogeneous paths exercise typed subcolumns with different numeric widths. + sql "DROP TABLE IF EXISTS variant_subfield_numeric" + sql """CREATE TABLE variant_subfield_numeric (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_subfield_numeric VALUES + (1, parse_to_variant('{"a":1,"b":1.0}')), + (2, parse_to_variant('{"a":1,"b":1.5}')), + (3, parse_to_variant('{"a":0,"b":-0.0}')), + (4, parse_to_variant('{"a":9007199254740993,"b":9007199254740992.0}')), + (5, parse_to_variant('{"a":9007199254740992,"b":9007199254740992.0}')), + (6, parse_to_variant('{"a":null,"b":1.0}')), + (7, parse_to_variant('{"a":1}')), + (8, parse_to_variant('{}')), + (9, NULL)""" + qt_typed_projection """SELECT id, v['a'], v['b'], + v['a'] = v['b'], v['a'] != v['b'], v['a'] <=> v['b'] + FROM variant_subfield_numeric ORDER BY id""" + ["=", "!=", "<=>"].each { op -> + qt_sql """SELECT id FROM variant_subfield_numeric + WHERE v['a'] ${op} v['b'] ORDER BY id""" + } + } +} diff --git a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy b/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy deleted file mode 100644 index 057166560d3531..00000000000000 --- a/regression-test/suites/variant_p0/test_variant_timestamptz_sparse.groovy +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Reproduces DORIS-25915: -// When VARIANT has typed paths that exceed variant_max_subcolumns_count and -// variant_enable_typed_paths_to_sparse=true, timestamptz values that fall to -// the sparse path are read back as only the timezone suffix (e.g. "+00:00") -// instead of the full timestamp. -suite("test_variant_timestamptz_sparse", "p0"){ - def variantV2Function = getFeConfig("enable_variant_v2").toBoolean() ? "parse_to_variant" : "" - sql " set time_zone = '+08:00' " - sql " set default_variant_enable_doc_mode = false " - - sql "DROP TABLE IF EXISTS test_variant_timestamptz_sparse_repro" - - sql """ - CREATE TABLE test_variant_timestamptz_sparse_repro ( - pk int, - var VARIANT< - 'c1':bigint, - 'c2':bigint, - 'd1':date, - 'd2':date, - 'dt0':datetime(0), - 'dt0n':datetime(0), - 'dt3':datetime(3), - 'dt3n':datetime(3), - 'dt6':datetime(6), - 'dt6n':datetime(6), - 'ts0':timestamptz(0), - 'ts0n':timestamptz(0), - 'ts3':timestamptz(3), - 'ts3n':timestamptz(3), - 'ts6':timestamptz(6), - 'ts6n':timestamptz(6), - 'v1':string, - 'v2':string, - properties("variant_max_subcolumns_count" = "2","variant_enable_typed_paths_to_sparse" = "true") - > NULL, - INDEX c1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c1"), - INDEX c2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "c2"), - INDEX d1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d1"), - INDEX d2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "d2"), - INDEX dt0 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0"), - INDEX dt0n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt0n"), - INDEX dt3 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3"), - INDEX dt3n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt3n"), - INDEX dt6 (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6"), - INDEX dt6n (`var`) USING INVERTED PROPERTIES("field_pattern" = "dt6n"), - INDEX v1 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v1"), - INDEX v2 (`var`) USING INVERTED PROPERTIES("field_pattern" = "v2") - ) ENGINE=OLAP - DUPLICATE KEY(pk) - DISTRIBUTED BY HASH(pk) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ - INSERT INTO test_variant_timestamptz_sparse_repro VALUES - (1, ${variantV2Function}('{"c1":1699516324,"c2":1690122421,"d1":"2024-02-25","d2":"2024-11-27","dt0":"2002-12-25 12:42:19","dt0n":"2015-02-05 02:36:13","dt3":"2015-02-22 02:09:01","dt3n":"2015-09-16 02:55:07","dt6":"2001-09-19 09:53:52","dt6n":"2003-12-21 02:29:00","ts0":"2003-12-21 02:29:00 +00:00","ts0n":"2003-12-21 02:29:00 +00:00","ts3":"2003-12-21 02:29:00 +00:00","ts3n":"2003-12-21 02:29:00 -05:00","ts6":"2003-12-21 02:29:00 -05:00","ts6n":"2003-12-21 02:29:00 +03:00","v1":"2024-12-10 10:16:19","v2":"2023-12-31 23:59:59"}')); - """ - - sql "SYNC" - - // Pre-fix: every column that fell to sparse returned just the timezone - // suffix ("+08:00"). Post-fix: each ts* path round-trips as a full timestamp. - qt_select_all """ - SELECT - pk, - CAST(CAST(var['ts0'] AS timestamptz(6)) AS string) AS str_ts0, - CAST(CAST(var['ts0n'] AS timestamptz(6)) AS string) AS str_ts0n, - CAST(CAST(var['ts3'] AS timestamptz(6)) AS string) AS str_ts3, - CAST(CAST(var['ts3n'] AS timestamptz(6)) AS string) AS str_ts3n, - CAST(CAST(var['ts6'] AS timestamptz(6)) AS string) AS str_ts6, - CAST(CAST(var['ts6n'] AS timestamptz(6)) AS string) AS str_ts6n - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ - - qt_select_cast_ts0 """ - SELECT - pk, - CAST(var['ts0'] AS timestamptz(0)) AS cast_ts0, - CAST(var['ts3'] AS timestamptz(3)) AS cast_ts3, - CAST(var['ts6'] AS timestamptz(6)) AS cast_ts6 - FROM test_variant_timestamptz_sparse_repro - ORDER BY pk; - """ -} diff --git a/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy b/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy new file mode 100644 index 00000000000000..8f467380e5ed33 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_v1_relational_compatibility.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_v1_relational_compatibility", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: false]) { + test { + sql ''' + SELECT v FROM ( + SELECT CAST('2' AS VARIANT) AS v + UNION ALL SELECT CAST('1' AS VARIANT) AS v + ) t ORDER BY v + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT v FROM ( + SELECT CAST('2' AS VARIANT) AS v + UNION ALL SELECT CAST('1' AS VARIANT) AS v + ) t ORDER BY v LIMIT 1 + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT row_number() OVER (ORDER BY parse_to_variant(CAST(number AS STRING))) + FROM numbers("number" = "2") + ''' + exception "Doris hll, bitmap" + } + test { + sql ''' + SELECT * FROM (SELECT CAST('1' AS VARIANT) AS v) a + JOIN (SELECT CAST('1' AS VARIANT) AS v) b ON a.v = b.v + ''' + exception "could not used in ComparisonPredicate" + } + order_qt_scalar_comparison ''' + SELECT CAST('1' AS VARIANT) = 1, CAST('2' AS VARIANT) > 1, + CAST(CAST('1' AS VARIANT) AS STRING) = '1' + ''' + order_qt_json_extract ''' + SELECT json_extract(CAST('{"id":1}' AS VARIANT), '$.id') + ''' + } +} diff --git a/regression-test/suites/variant_p0/test_variant_window_partition.groovy b/regression-test/suites/variant_p0/test_variant_window_partition.groovy new file mode 100644 index 00000000000000..0860c2537423e9 --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_window_partition.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// GROUP BY, ORDER BY, hash join and window ORDER BY accept Variant, and they all use the same +// canonical equality and comparison. Window PARTITION BY used to be the one remaining context that +// rejected it, so these queries pin that a Variant partition key groups rows by canonical value: +// values of different JSON kinds ("a", true, 1) never share a partition, and equal values do. +suite("test_variant_window_partition", "p0,nonConcurrent") { + setFeConfigTemporary([enable_variant_v2: true]) { + + sql "drop table if exists test_variant_window_partition" + + sql """ + create table test_variant_window_partition ( + id int, + v variant + ) engine = olap + duplicate key (id) + distributed by hash(id) buckets 1 + properties ("replication_num" = "1") + """ + + sql """ + insert into test_variant_window_partition values + (1, parse_to_variant('{"k": 1}')), + (2, parse_to_variant('{"k": 1}')), + (3, parse_to_variant('{"k": 2}')), + (4, parse_to_variant('{"k": "a"}')), + (5, parse_to_variant('{"k": true}')), + (6, parse_to_variant('{"k": true}')), + (7, parse_to_variant('{"k": 1}')), + (8, parse_to_variant('{"k": true}')) + """ + + // PARTITION BY a Variant sub-path. + order_qt_partition_by_subpath """ + select id, v['k'] as k, row_number() over (partition by v['k'] order by id) as rn + from test_variant_window_partition + """ + + // PARTITION BY the whole Variant value. + order_qt_partition_by_whole """ + select id, row_number() over (partition by v order by id) as rn + from test_variant_window_partition + """ + + // rank() with a Variant partition key. + order_qt_partition_rank """ + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + """ + + // CreatePartitionTopNFromWindow matches logicalFilter(logicalWindow()), so a PartitionTopN + // operator only appears when a filter sits above the window. That operator organises the + // Variant partition key as a serialized hash key on BE, which the queries above never reach. + order_qt_partition_topn """ + select id, rk from ( + select id, rank() over (partition by v['k'] order by id) as rk + from test_variant_window_partition + ) t where rk <= 2 + """ + + // A Variant partition key combined with a scalar one. + order_qt_partition_mixed_keys """ + select id, row_number() over (partition by v['k'], id % 2 order by id) as rn + from test_variant_window_partition + """ + + // Once the analytic sink has buffered more than 256 input blocks it erases the rows it has + // consumed from its partition and order key columns. A small batch size reaches that with a + // few thousand rows; the keys are typed Variant (CAST of a number) and encoded Variant + // (parse_to_variant) so both layouts are erased. + sql "set batch_size = 8" + qt_many_blocks_typed_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select cast(number as variant) v, cast(number % 3 as variant) p + from numbers("number" = "5000")) t + ) r + """ + qt_many_blocks_encoded_keys """ + select count(*), sum(rn), sum(dr) from ( + select row_number() over (order by v) as rn, + dense_rank() over (partition by p order by v) as dr + from (select parse_to_variant(cast(number as string)) v, + parse_to_variant(cast(number % 3 as string)) p + from numbers("number" = "5000")) t + ) r + """ + } +} diff --git a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy index 9e8afe7ea14fcb..bd8987819856db 100644 --- a/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy +++ b/regression-test/suites/variant_p0/v2/test_variant_equality_contexts.groovy @@ -77,23 +77,14 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { contains "EXCEPT" } - test { - sql "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + // V2 canonical equality is supported in scalar and join contexts. + qt_equals "SELECT ${variantV2Function}('1') = ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_not_equals "SELECT ${variantV2Function}('1') != ${variantV2Function}('1.0')" - test { - sql "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - exception "CAST to a concrete type first" - } + qt_null_safe_equals "SELECT ${variantV2Function}('1') <=> ${variantV2Function}('1.0')" - test { - sql """ + order_qt_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -101,11 +92,8 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v = b.v """ - exception "CAST to a concrete type first" - } - test { - sql """ + order_qt_null_safe_equality_join """ SELECT * FROM (SELECT CAST(CAST(number AS STRING) AS VARIANT) v FROM numbers("number" = "2")) a @@ -113,8 +101,6 @@ suite("test_variant_equality_contexts", "p0,nonConcurrent") { FROM numbers("number" = "2")) b ON a.v <=> b.v """ - exception "CAST to a concrete type first" - } order_qt_group_by """ SELECT CAST(v AS STRING), COUNT(*) diff --git a/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md new file mode 100644 index 00000000000000..1c2741280d3081 --- /dev/null +++ b/regression-test/suites/variant_p2/RELATIONAL_BENCHMARK.md @@ -0,0 +1,101 @@ +# VARIANT relational benchmark + +On branch-4.2, the benchmark load and relational suites temporarily enable +Variant V2 and restore the previous FE configuration after each phase. A plain +`load_p2` run keeps the current Variant configuration. + +`relational_performance.groovy` extends the existing `variant_p2` GitHub Events +workload with GROUP BY, ORDER BY, and broadcast/shuffle equality JOIN. The load +phase runs `load.groovy` and verifies its expected 44,273,863 rows. Pass +`--stream-load` to read the same public hourly files over HTTPS when regression +S3 credentials are unavailable. HTTP reads use a socket timeout and resumable +download. If an interrupted run has committed all but a known set of files, +pass those names with `--resume-files`; this appends only those files before the +same exact row-count check and prepare phase. +It then creates one small dimension table per selected real JSON path: + +| Key | Source path | CAST comparison | +| --- | --- | --- | +| actor_login | `actor['login']` | STRING | +| repo_name | `repo['name']` | STRING | +| payload_action | `payload['action']` | STRING | +| actor_id | `actor['id']` | BIGINT | + +The default pair is `actor_login,actor_id`, covering one string and one numeric +path. Use `--keys` to select any subset of the four paths. + +Each query has native VARIANT and explicit CAST forms. SQL cache and query cache +are disabled. GROUP BY correctness uses bidirectional EXCEPT between complete +native and CAST group results before timing compact aggregate checksums. JOIN +checks count and ID sum and uses one dimension row per key to avoid many-to-many +output explosion. + +| Operation | What it measures | +| --- | --- | +| `group` | GROUP BY on the key, reduced to a compact checksum | +| `order_topn` | `ORDER BY key NULLS FIRST, id LIMIT 1000`: a TopN, not a full sort | +| `order_full` | `row_number() OVER (ORDER BY key NULLS FIRST, id)` over every row: a full sort whose `sum(id * rn)` checksum compares the complete native and CAST orders | +| `join_broadcast`, `join_shuffle` | equality join against the dimension table | + +Variant join keys get no runtime filter while CAST keys do. Each join therefore +has three modes: `native`, `cast`, and `cast_no_rf` (the CAST join with +`runtime_filter_mode='OFF'`). Compare `native` with `cast_no_rf` for the key +comparison cost and `cast` with `cast_no_rf` for the runtime filter benefit. + +A join against an empty or stale dimension table would still make native and +CAST agree. Before timing a join, the query phase checks with bidirectional +EXCEPT that the dimension table equals the current grouping of `github_events` +and is not empty; rerun the load phase (or `VARIANT_BENCH_PHASE=prepare`) if it +fails. + +Mixed-type semantics belong in `variant_p0/test_variant_relational_corners.groovy`. +They are deliberately excluded from these performance pairs: CAST may merge +strings with numbers or round integers above 2^53. Matching CAST results alone +cannot prove correct native mixed-type behavior. + +Build and start this worktree's FE/BE using the repository environment workflow. +Performance requires `BUILD_TYPE=RELEASE` and `./build.sh --be --fe`; do not pass +an explicit `-j`. Synchronize worktree ports after building. The runner requires +Python 3.11+, Linux `/proc`, and the local `output/` runtime. Use a dedicated +database/config because `load.groovy` recreates `github_events`. + +Run from the worktree root, substituting the verified regression config: + +```bash +python3 regression-test/suites/variant_p2/run_relational_benchmark.py load \ + --conf tmp/regression-conf.auto.groovy --stream-load \ + --output tmp/relational-load + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --warmups 2 --repeats 7 \ + --output tmp/relational-query + +python3 regression-test/suites/variant_p2/run_relational_benchmark.py query \ + --conf tmp/regression-conf.auto.groovy \ + --cpus 16,17,18,19,20,21,22,23 --keys actor_login \ + --warmups 0 --repeats 3 --spill --output tmp/relational-spill +``` + +Choose eight distinct physical cores after inspecting host topology and load. +The load phase does not change affinity and rejects an already eight-core-bound +FE or BE process. Doris may independently pin a few service threads. The query +phase binds every FE and BE thread to the same eight cores, checks affinity +during the run, and restores previous masks afterward. This restricts the +database servers; the regression client runs outside that mask. CPU affinity +does not reserve cores against unrelated host processes. + +The runner verifies the running BE matches the installed Release binary. Each +new evidence directory contains revision/binary/affinity metadata, host load, +regression logs, query plans, every warmup/measured latency and result SHA-256, +and median/min/max summaries. Latency includes client round-trip and result +consumption. The starting mode rotates each round. Interpret forced-spill +measurements separately from ordinary queries. A completed repeat run checks +repeatability, not long-duration soak, concurrent ingestion, or crash recovery. + +The query suite can also run directly through `run-regression-test.sh` using +`VARIANT_BENCH_PHASE=prepare|query` (required: without it an ordinary `variant_p2` +run skips the suite), `VARIANT_BENCH_ROWS`, `VARIANT_BENCH_KEYS`, +`VARIANT_BENCH_REPEATS`, `VARIANT_BENCH_WARMUPS`, `VARIANT_BENCH_RESULTS`, and +`VARIANT_BENCH_SPILL`. Direct runs do not enforce Release or CPU affinity and +must not be reported as an eight-core performance result. diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 8cb0bc19255f30..2e1f8b593527fb 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -17,163 +17,177 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -suite("load_p2", "variant_type,p2"){ - boolean use_stream_load = false - def load_json_data = {table_name, file_name -> - // load the json data - streamLoad { - table "${table_name}" +suite("load_p2", "variant_type,p2,nonConcurrent"){ + def enableVariantV2 = System.getenv().containsKey("VARIANT_BENCH_PHASE") || + getFeConfig("enable_variant_v2").toBoolean() + setFeConfigTemporary([enable_variant_v2: enableVariantV2]) { + def env = System.getenv() + boolean use_stream_load = env + .getOrDefault("VARIANT_P2_USE_STREAM_LOAD", "false").toBoolean() + def resumeFiles = env.getOrDefault("VARIANT_P2_RESUME_FILES", "") + .split(",").findAll { !it.isEmpty() } + def load_json_data = {table_name, file_name -> + // load the json data + streamLoad { + table "${table_name}" - // set http request header params - set 'read_json_by_line', 'true' - set 'format', 'json' - set 'max_filter_ratio', '0.1' - file file_name // import json file - time 10000 // limit inflight 10s + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + set 'max_filter_ratio', '0.1' + file file_name // import json file + time 10000 // limit inflight 10s + timeout 60000 + retryIfHttpError true - // if declared a check callback, the default check condition will ignore. - // So you must check all condition + // if declared a check callback, the default check condition will ignore. + // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Stream load ${file_name} result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } - logger.info("Stream load ${file_name} result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } } - } - def create_table = {table_name, buckets="auto" -> - sql "set default_variant_max_subcolumns_count = 0" - sql "DROP TABLE IF EXISTS ${table_name}" - sql """ - CREATE TABLE IF NOT EXISTS ${table_name} ( - id BIGINT NOT NULL, - type VARCHAR(30) NULL, - actor VARIANT NULL, - repo VARIANT NULL, - payload VARIANT NULL, - public BOOLEAN NULL, - created_at DATETIME NULL, - org JSON NULL - -- INDEX idx_payload(payload) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', - -- INDEX idx_repo(repo) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', - -- INDEX idx_actor(actor) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' - ) - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(id) BUCKETS ${buckets} - properties("replication_num" = "1", "disable_auto_compaction" = "false"); - """ - } + def create_table = {table_name, buckets="auto" -> + sql "set default_variant_max_subcolumns_count = 0" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + id BIGINT NOT NULL, + type VARCHAR(30) NULL, + actor VARIANT NULL, + repo VARIANT NULL, + payload VARIANT NULL, + public BOOLEAN NULL, + created_at DATETIME NULL, + org JSON NULL + -- INDEX idx_payload(payload) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', + -- INDEX idx_repo(repo) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', + -- INDEX idx_actor(actor) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(id) BUCKETS ${buckets} + properties("replication_num" = "1", "disable_auto_compaction" = "false"); + """ + } - // Configuration for the number of threads - def numberOfThreads = 20 // Set this to your desired number of threads + // Configuration for the number of threads + def numberOfThreads = 20 // Set this to your desired number of threads - // Executor service for managing threads - def executorService = Executors.newFixedThreadPool(numberOfThreads) - def futures = [] + // Executor service for managing threads + def executorService = Executors.newFixedThreadPool(numberOfThreads) + def futures = [] - try { - def table_name = "github_events" - def s3load_paral_wait = {tbl, fmt, path, paral -> - String ak = getS3AK() - String sk = getS3SK() - String s3BucketName = getS3BucketName() - String s3Endpoint = getS3Endpoint() - String s3Region = getS3Region() - def load_label = "part_" + UUID.randomUUID().toString().replace("-", "0") - sql """ - LOAD LABEL ${load_label} ( - DATA INFILE("s3://${s3BucketName}/${path}") - INTO TABLE ${tbl} - COLUMNS TERMINATED BY "," - FORMAT AS "${fmt}" - ) - WITH S3 ( - "AWS_ACCESS_KEY" = "$ak", - "AWS_SECRET_KEY" = "$sk", - "AWS_ENDPOINT" = "${s3Endpoint}", - "AWS_REGION" = "${s3Region}", - "provider" = "${getS3Provider()}" - ) - PROPERTIES( - "load_parallelism" = "${paral}" - ); - """ - // Waiting for job finished or cancelled - def max_try_milli_secs = 600000 - while (max_try_milli_secs > 0) { - String[][] result = sql """ show load where label="$load_label" order by createtime desc limit 1; """ - if (result[0][2].equals("FINISHED")) { - logger.info("Load FINISHED " + load_label) - break; - } - if (result[0][2].equals("CANCELLED")) { - log.warn("load failed: $result") - assertTrue(false, "load failed: $result") - break; + try { + def table_name = "github_events" + def s3load_paral_wait = {tbl, fmt, path, paral -> + String ak = getS3AK() + String sk = getS3SK() + String s3BucketName = getS3BucketName() + String s3Endpoint = getS3Endpoint() + String s3Region = getS3Region() + def load_label = "part_" + UUID.randomUUID().toString().replace("-", "0") + sql """ + LOAD LABEL ${load_label} ( + DATA INFILE("s3://${s3BucketName}/${path}") + INTO TABLE ${tbl} + COLUMNS TERMINATED BY "," + FORMAT AS "${fmt}" + ) + WITH S3 ( + "AWS_ACCESS_KEY" = "$ak", + "AWS_SECRET_KEY" = "$sk", + "AWS_ENDPOINT" = "${s3Endpoint}", + "AWS_REGION" = "${s3Region}", + "provider" = "${getS3Provider()}" + ) + PROPERTIES( + "load_parallelism" = "${paral}" + ); + """ + // Waiting for job finished or cancelled + def max_try_milli_secs = 600000 + while (max_try_milli_secs > 0) { + String[][] result = sql """ show load where label="$load_label" order by createtime desc limit 1; """ + if (result[0][2].equals("FINISHED")) { + logger.info("Load FINISHED " + load_label) + break; + } + if (result[0][2].equals("CANCELLED")) { + log.warn("load failed: $result") + assertTrue(false, "load failed: $result") + break; + } + Thread.sleep(6000) + max_try_milli_secs -= 6000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $load_label") + } } - Thread.sleep(6000) - max_try_milli_secs -= 6000 - if(max_try_milli_secs <= 0) { - assertTrue(1 == 2, "load Timeout: $load_label") + } + def submitFile = { fileName -> + if (use_stream_load) { + def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + load_json_data.call(table_name, fileUrl) + } as Runnable) + } else { + futures << executorService.submit({ + log.info("Loading file: ${fileName}") + s3load_paral_wait.call(table_name, "JSON", + "regression/github_events_dataset/${fileName}", 3) + } as Runnable) } } - } - create_table.call(table_name) - List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - // 2015 - def year = "2015" - def monthPrefix = "0" - def dayPrefix = "0" - log.info("current year: ${year}") - for (int i = 1; i <= 3; i++) { - def month = i < 10 ? monthPrefix + i.toString() : i.toString() - log.info("current month: ${month}") - for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { - def day = j < 10 ? dayPrefix + j.toString() : j.toString() - log.info("current day: ${day}") - for (int z = 1; z < 24; z++) { - def hour = z.toString() - log.info("current hour: ${hour}") - def fileName = year + "-" + month + "-" + day + "-" + hour + ".json" - log.info("cuurent fileName: ${fileName}") - if (use_stream_load) { - def fileUrl = """${getS3Url() + '/regression/github_events_dataset/' + fileName}""" - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - load_json_data.call(table_name, fileUrl) - } as Runnable) - } else { - // Submitting tasks to the executor service - futures << executorService.submit({ - log.info("Loading file: ${fileName}") - s3load_paral_wait.call(table_name, "JSON", "regression/github_events_dataset/${fileName}", 3) - } as Runnable) + if (resumeFiles) { + assertTrue(use_stream_load, "Resume requires VARIANT_P2_USE_STREAM_LOAD=true") + resumeFiles.each { submitFile.call(it) } + } else { + create_table.call(table_name) + List daysEveryMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + def year = "2015" + def monthPrefix = "0" + def dayPrefix = "0" + log.info("current year: ${year}") + for (int i = 1; i <= 3; i++) { + def month = i < 10 ? monthPrefix + i.toString() : i.toString() + log.info("current month: ${month}") + for (int j = 1; j <= daysEveryMonth[i - 1]; j++) { + def day = j < 10 ? dayPrefix + j.toString() : j.toString() + log.info("current day: ${day}") + for (int z = 1; z < 24; z++) { + def fileName = year + "-" + month + "-" + day + "-" + z + ".json" + log.info("current fileName: ${fileName}") + submitFile.call(fileName) + } } } } - } - - try { - futures.each { future -> - future.get() + + try { + futures.each { future -> + future.get() + } + } catch (Exception e) { + throw e.cause // throw original exception + } finally { + // Shutdown executor service and wait for all tasks to complete + executorService.shutdown() + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) } - } catch (Exception e) { - throw e.cause // throw original exception + + qt_sql("select count() from github_events") } finally { - // Shutdown executor service and wait for all tasks to complete - executorService.shutdown() - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) + // reset flags } - - qt_sql("select count() from github_events") - } finally { - // reset flags } } diff --git a/regression-test/suites/variant_p2/relational_performance.groovy b/regression-test/suites/variant_p2/relational_performance.groovy new file mode 100644 index 00000000000000..f9355556b9e13b --- /dev/null +++ b/regression-test/suites/variant_p2/relational_performance.groovy @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import java.security.MessageDigest + +suite("variant_relational_performance", "p2,nonConcurrent") { + def env = System.getenv() + // This benchmark is driven by run_relational_benchmark.py. A plain variant_p2 run has no + // prepared dimension tables, so the suite only runs when a phase is set explicitly. + def phase = env.get("VARIANT_BENCH_PHASE") + if (phase == null) { + log.info("Skip variant_relational_performance: VARIANT_BENCH_PHASE is not set") + return + } + long expectedRows = env.getOrDefault("VARIANT_BENCH_ROWS", "44273863").toLong() + int repeats = env.getOrDefault("VARIANT_BENCH_REPEATS", "7").toInteger() + int warmups = env.getOrDefault("VARIANT_BENCH_WARMUPS", "2").toInteger() + if (!(phase in ["prepare", "query"]) || expectedRows < 1 || repeats < 1 || warmups < 0) { + throw new IllegalArgumentException("Invalid VARIANT_BENCH configuration") + } + // These paths come directly from variant_p2/sql and the GitHub Events schema. + def keys = [ + actor_login: [column: "actor", path: "login", type: "STRING"], + repo_name: [column: "repo", path: "name", type: "STRING"], + payload_action: [column: "payload", path: "action", type: "STRING"], + actor_id: [column: "actor", path: "id", type: "BIGINT"] + ] + def requestedKeys = env.getOrDefault("VARIANT_BENCH_KEYS", "actor_login,actor_id").split(",") as Set + if (!keys.keySet().containsAll(requestedKeys)) { + throw new IllegalArgumentException("Unknown VARIANT_BENCH_KEYS: ${requestedKeys - keys.keySet()}") + } + keys = keys.findAll { key, ignored -> requestedKeys.contains(key) } + + setFeConfigTemporary([enable_variant_v2: true]) { + def actualRows = (sql("SELECT count(*) FROM github_events"))[0][0].toString().toLong() + assertEquals(expectedRows, actualRows) + if (phase == "prepare") { + sql "SET default_variant_max_subcolumns_count = 0" + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + sql "DROP TABLE IF EXISTS variant_relational_dim_${key}" + sql """CREATE TABLE variant_relational_dim_${key} (id BIGINT, k VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO variant_relational_dim_${key} + SELECT min(id), ${nativeKey} FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${nativeKey}""" + } + return + } + + sql "SET parallel_pipeline_task_num = 8" + sql "SET enable_sql_cache = false" + sql "SET enable_query_cache = false" + if (env.getOrDefault("VARIANT_BENCH_SPILL", "false").toBoolean()) { + sql "SET enable_spill = true" + sql "SET enable_force_spill = true" + sql "SET spill_min_revocable_mem = 1" + } + def output = new File(env.getOrDefault("VARIANT_BENCH_RESULTS", + "tmp/variant-relational-results.jsonl")) + output.parentFile.mkdirs() + def record = { event -> output.append(JsonOutput.toJson(event) + "\n") } + def fingerprint = { result -> + MessageDigest.getInstance("SHA-256").digest(JsonOutput.toJson( + result.collect { row -> row.collect { value -> value == null ? null : value.toString() } } + ).getBytes("UTF-8")).encodeHex().toString() + } + def query = { statement -> + long start = System.nanoTime() + def result = sql(statement) + [ms: (System.nanoTime() - start) / 1e6, hash: fingerprint(result), resultRows: result.size()] + } + record([event: "start", rows: actualRows, repeats: repeats, warmups: warmups, + parallelPipelineTasks: 8, keys: keys.keySet(), cpus: env.get("VARIANT_BENCH_CPUS"), + spill: env.get("VARIANT_BENCH_SPILL"), time: new Date().toString()]) + keys.each { key, spec -> + def nativeKey = "${spec.column}['${spec.path}']" + def castKey = "CAST(${nativeKey} AS ${spec.type})" + def nativeGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${nativeKey}""" + def castGroups = """SELECT ${castKey} k, min(id) first_id, count(*) n + FROM github_events GROUP BY ${castKey}""" + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${nativeGroups}) EXCEPT (${castGroups})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${castGroups}) EXCEPT (${nativeGroups})) difference"""))[0][0].toString().toInteger()) + record([event: "full_group_correctness", key: key]) + + // ORDER BY ... LIMIT is a TopN and never sorts the whole input. The window forces a + // full sort of every row, and its order-sensitive checksum compares the complete + // native order with the CAST order instead of the first rows only. + def fullSort = { orderKey -> """SELECT count(*), sum(CAST(id AS LARGEINT) * rn) FROM ( + SELECT id, row_number() OVER (ORDER BY ${orderKey} NULLS FIRST, id) rn + FROM github_events) ranked""" } + def queries = [ + group: [ + native: "SELECT count(*), sum(n*n), sum(first_id) FROM (${nativeGroups}) g", + cast: "SELECT count(*), sum(n*n), sum(first_id) FROM (${castGroups}) g" + ], + order_topn: [ + native: "SELECT id FROM github_events ORDER BY ${nativeKey} NULLS FIRST, id LIMIT 1000", + cast: "SELECT id FROM github_events ORDER BY ${castKey} NULLS FIRST, id LIMIT 1000" + ], + order_full: [native: fullSort(nativeKey), cast: fullSort(castKey)] + ] + + // A join against an empty or stale dimension still makes native and CAST agree, so + // prove the dimension is the current grouping of the source before timing a join. + def dimension = "SELECT id, CAST(k AS ${spec.type}) k FROM variant_relational_dim_${key}" + def expectedDimension = """SELECT min(id) id, ${castKey} k FROM github_events + WHERE ${nativeKey} IS NOT NULL GROUP BY ${castKey}""" + def dimensionRows = (sql("SELECT count(*) FROM variant_relational_dim_${key}"))[0][0] + .toString().toLong() + assertTrue(dimensionRows > 0) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${dimension}) EXCEPT (${expectedDimension})) difference"""))[0][0].toString().toInteger()) + assertEquals(0, (sql("""SELECT count(*) FROM ( + (${expectedDimension}) EXCEPT (${dimension})) difference"""))[0][0].toString().toInteger()) + record([event: "dimension_verified", key: key, rows: dimensionRows]) + + // Variant keys get no runtime filter while the CAST keys do, so cast_no_rf isolates the + // key comparison cost from the runtime filter benefit. + def leftKey = "l.${spec.column}['${spec.path}']" + def castPredicate = "CAST(${leftKey} AS ${spec.type}) = CAST(r.k AS ${spec.type})" + ["broadcast", "shuffle"].each { distribution -> + def join = { hint, predicate -> """SELECT ${hint} count(*), sum(l.id) FROM github_events l + JOIN [${distribution}] variant_relational_dim_${key} r ON ${predicate}""" } + queries["join_${distribution}".toString()] = [ + native: join("", "${leftKey} = r.k"), + cast: join("", castPredicate), + cast_no_rf: join("/*+ SET_VAR(runtime_filter_mode='OFF') */", castPredicate) + ] + } + + queries.each { operation, statements -> + def oracle = query(statements.cast).hash + statements.each { mode, statement -> + record([event: "plan", key: key, operation: operation, mode: mode, + plan: sql("EXPLAIN ${statement}")]) + } + def modes = statements.keySet() as List + for (int round = -warmups; round < repeats; ++round) { + // Rotate the starting mode so no mode always runs first or last. + Collections.rotate(modes, 1) + modes.each { mode -> + def sample = query(statements[mode]) + assertEquals(oracle, sample.hash) + record(sample + [event: "sample", key: key, operation: operation, + mode: mode, round: round, sql: statements[mode]]) + } + } + } + } + record([event: "complete", time: new Date().toString()]) + } +} diff --git a/regression-test/suites/variant_p2/run_relational_benchmark.py b/regression-test/suites/variant_p2/run_relational_benchmark.py new file mode 100644 index 00000000000000..72e9a19660dd89 --- /dev/null +++ b/regression-test/suites/variant_p2/run_relational_benchmark.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run the regression suite with unrestricted load and verified 8-core queries.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import statistics +import subprocess +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("load", "query")) + parser.add_argument("--conf", required=True) + parser.add_argument("--cpus", help="Eight comma-separated physical CPU IDs (query only)") + parser.add_argument("--rows", type=int, default=44_273_863) + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--keys", default="actor_login,actor_id") + parser.add_argument("--stream-load", action="store_true", + help="Load public variant_p2 files over HTTP instead of authenticated S3") + parser.add_argument("--resume-files", default="", + help="Comma-separated public variant_p2 files to append after an interrupted load") + parser.add_argument("--spill", action="store_true", help="Separate forced-spill correctness/stability run") + parser.add_argument("--output", required=True, help="New evidence directory") + args = parser.parse_args() + if args.phase == "query" and (args.stream_load or args.resume_files): + parser.error("--stream-load and --resume-files apply only to the load phase") + if args.resume_files and not args.stream_load: + parser.error("--resume-files requires --stream-load") + # Job cancellation sends SIGTERM. Unwind through the finally block below so the regression + # child is stopped and FE/BE thread affinity is restored. + signal.signal(signal.SIGTERM, lambda signum, _: sys.exit(128 + signum)) + repo = Path(__file__).resolve().parents[3] + evidence = Path(args.output).resolve() + evidence.mkdir(parents=True, exist_ok=False) + processes = {} + original = {} + for name in ("be", "fe"): + pid = int((repo / f"output/{name}/bin/{name}.pid").read_text()) + command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode() + if str(repo) not in command: + raise RuntimeError(f"Refusing to change unrelated {name} PID {pid}") + processes[name] = pid + original[pid] = {int(t.name): os.sched_getaffinity(int(t.name)) + for t in Path(f"/proc/{pid}/task").iterdir()} + # The regression client must query the FE whose threads are pinned and fingerprinted here. + conf_port = re.search(r"jdbc:mysql://[^:/]+:(\d+)", Path(args.conf).read_text()) + fe_port = re.search(r"^\s*query_port\s*=\s*(\d+)", + (repo / "output/fe/conf/fe.conf").read_text(), re.MULTILINE) + if conf_port is None or int(conf_port.group(1)) != int(fe_port.group(1) if fe_port else 9030): + raise RuntimeError("--conf does not point at this worktree's FE query port") + cpus = None + if args.phase == "query": + cpus = {int(cpu) for cpu in (args.cpus or "").split(",") if cpu} + topology = {tuple((Path(f"/sys/devices/system/cpu/cpu{cpu}/topology") / item) + .read_text().strip() for item in ("physical_package_id", "core_id")) + for cpu in cpus} + if len(cpus) != 8 or len(topology) != 8: + raise ValueError("Select exactly eight distinct physical cores") + cache = (repo / "be/build_RELEASE/CMakeCache.txt").read_text() + if "CMAKE_BUILD_TYPE:STRING=RELEASE" not in cache.upper(): + raise RuntimeError("A Release build is required for performance measurements") + binary = repo / "output/be/lib/doris_be" + running = Path(f"/proc/{processes['be']}/exe") + if not os.path.samefile(binary, running): + raise RuntimeError("Running BE differs from the worktree output binary") + release_binary = repo / "be/build_RELEASE/src/service/doris_be" + with release_binary.open("rb") as built, binary.open("rb") as installed: + if hashlib.file_digest(built, "sha256").digest() != hashlib.file_digest(installed, "sha256").digest(): + raise RuntimeError("Installed BE does not match the Release build") + elif args.cpus: + raise ValueError("Do not bind CPUs during ingestion") + else: + for pid in processes.values(): + if len(os.sched_getaffinity(pid)) <= 8: + raise RuntimeError("Load requires each FE/BE process to be unrestricted beyond eight CPUs") + manifest = dict(vars(args), processes=processes, checkout=str(repo), + head=subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip(), + started=time.time(), original_affinity={str(p): {str(t): sorted(m) for t, m in ts.items()} + for p, ts in original.items()}) + manifest["harness_sha256"] = { + name: hashlib.sha256((Path(__file__).parent / name).read_bytes()).hexdigest() + for name in ("load.groovy", "relational_performance.groovy", Path(__file__).name) + } + manifest["dirty_checkout"] = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], cwd=repo, text=True).strip()) + # Hash what the recorded processes run, not only what the checkout contains. + with Path(f"/proc/{processes['be']}/exe").open("rb") as binary: + manifest["be_sha256"] = hashlib.file_digest(binary, "sha256").hexdigest() + with (repo / "output/fe/lib/doris-fe.jar").open("rb") as jar: + manifest["fe_jar_sha256"] = hashlib.file_digest(jar, "sha256").hexdigest() + (evidence / "manifest.json").write_text(json.dumps(manifest, indent=2)) + environment = dict(os.environ, VARIANT_BENCH_PHASE=args.phase, + VARIANT_BENCH_ROWS=str(args.rows), VARIANT_BENCH_REPEATS=str(args.repeats), + VARIANT_BENCH_WARMUPS=str(args.warmups), + VARIANT_BENCH_KEYS=args.keys, + VARIANT_BENCH_RESULTS=str(evidence / "samples.jsonl"), + VARIANT_BENCH_SPILL=str(args.spill).lower(), + VARIANT_P2_USE_STREAM_LOAD=str(args.stream_load).lower(), + VARIANT_P2_RESUME_FILES=args.resume_files, + VARIANT_BENCH_CPUS=args.cpus or "unrestricted") + for key in list(environment): + if key.lower() in ("http_proxy", "https_proxy", "all_proxy"): + del environment[key] + environment["NO_PROXY"] = environment["no_proxy"] = "127.0.0.1,localhost" + child = None + try: + if cpus: + # Re-enumerate until every thread has the mask. New threads inherit + # the creator's mask; the monitor below rejects any subsequent drift. + for pid in processes.values(): + for _ in range(10): + tids = [int(t.name) for t in Path(f"/proc/{pid}/task").iterdir()] + for tid in tids: + try: + os.sched_setaffinity(tid, cpus) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(t.name)) == cpus + for t in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not establish CPU affinity") + suites = (["load_p2", "variant_relational_performance"] + if args.phase == "load" else ["variant_relational_performance"]) + with (evidence / "host-load.jsonl").open("w") as host: + for suite in suites: + suite_environment = environment.copy() + if args.phase == "load" and suite == "variant_relational_performance": + suite_environment["VARIANT_BENCH_PHASE"] = "prepare" + log_name = ({"load_p2": "load.log"}.get(suite, "prepare.log") + if args.phase == "load" else "regression.log") + command = [str(repo / "run-regression-test.sh"), "--conf", args.conf, "--run", + "-d", "variant_p2", "-s", suite] + with (evidence / log_name).open("w") as log: + child = subprocess.Popen(command, cwd=repo, env=suite_environment, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + while child.poll() is None: + host.write(json.dumps({"time": time.time(), + "loadavg": Path("/proc/loadavg").read_text().strip()}) + "\n") + host.flush() + if cpus: + for pid in processes.values(): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + if os.sched_getaffinity(int(thread.name)) != cpus: + raise RuntimeError(f"CPU affinity drift: {thread}") + except ProcessLookupError: + pass + time.sleep(1) + if child.returncode: + raise RuntimeError(f"Regression failed: inspect {evidence / log_name}") + child = None + finally: + if child is not None and child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + child.wait() + if cpus: + for pid, threads in original.items(): + for _ in range(10): + for thread in Path(f"/proc/{pid}/task").iterdir(): + try: + os.sched_setaffinity(int(thread.name), threads.get(int(thread.name), threads[pid])) + except ProcessLookupError: + pass + if all(os.sched_getaffinity(int(thread.name)) == + threads.get(int(thread.name), threads[pid]) + for thread in Path(f"/proc/{pid}/task").iterdir()): + break + else: + raise RuntimeError("Could not restore CPU affinity") + if args.phase == "query": + events = [json.loads(line) for line in (evidence / "samples.jsonl").read_text().splitlines()] + if not events or events[-1]["event"] != "complete": + raise RuntimeError("Missing completion record") + groups = {} + for event in events: + if event["event"] == "sample" and event["round"] >= 0: + groups.setdefault((event["key"], event["operation"], event["mode"]), []).append(event["ms"]) + summary = [{"key": key, "operation": operation, "mode": mode, "samples_ms": values, + "median_ms": statistics.median(values), "min_ms": min(values), "max_ms": max(values)} + for (key, operation, mode), values in groups.items()] + (evidence / "summary.json").write_text(json.dumps(summary, indent=2)) + print(evidence) + + +if __name__ == "__main__": + main()