Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 203 additions & 2 deletions be/src/core/column/variant_v2/column_variant_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
#include "core/value/variant/variant_canonical.h"
#include "core/value/variant/variant_field.h"
#include "core/value/variant/variant_parquet_encoding.h"
#include "exec/sort/sort_block.h"

namespace doris {
namespace {
Expand Down Expand Up @@ -689,6 +690,33 @@ const IColumn& ColumnVariantV2::typed_column() const {
return *_typed;
}

void ColumnVariantV2::validate_typed_rows() const {
DORIS_CHECK(_typed != nullptr) << "validate_typed_rows requires ColumnVariantV2 typed state";
switch (_typed_type->get_primitive_type()) {
// Every value of these types maps to a Variant scalar, so there is nothing to reject and no
// reason to format each LARGEINT or IP address only to discard the text.
case TYPE_BOOLEAN:
case TYPE_TINYINT:
case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_BIGINT:
case TYPE_LARGEINT:
case TYPE_FLOAT:
case TYPE_DOUBLE:
case TYPE_IPV4:
case TYPE_IPV6:
return;
default:
break;
}
// with_variant_typed_scalar() is the one definition of what Variant can encode: it throws for
// an invalid date, a string that is not UTF-8 or a decimal outside its width. A type missing
// from the list above is validated, which is the safe side.
visit_typed_scalar_column(assert_cast<const ColumnNullable&>(*_typed),
_typed_type->get_primitive_type(), _typed_type->get_scale(), 0,
_typed->size(), [](size_t, const VariantScalarRef&) {});
}

const DataTypePtr& ColumnVariantV2::typed_type() const {
DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state";
return _typed_type;
Expand Down Expand Up @@ -1285,6 +1313,132 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row) const {
return {.metadata = {.data = metadata.data, .size = metadata.size}, .value = value};
}

std::pair<const ColumnNullable*, const ColumnNullable*> ColumnVariantV2::_typed_ordering_operands(
const ColumnVariantV2& right, bool allow_floating) const {
if (!is_typed() || !right.is_typed() ||
!(_typed_type == right._typed_type || _typed_type->equals(*right._typed_type))) {
return {nullptr, nullptr};
}
// Admission is a whitelist: a typed type takes the native path only after its native order has
// been checked against the canonical Variant order (scalar_compare in variant_canonical.cpp).
// The premises shared by every admitted type:
// - both sides have the same typed type including its parameters, so DECIMAL scales and
// DATETIME precisions match and native values are directly comparable;
// - every non-null typed row is a value Variant can encode (the rule and who enforces it
// are at create_typed()), so the native order does not rank a value the canonical path
// would reject, such as an invalid date or a string that is not UTF-8;
// - a typed NULL is the Variant null, which the callers order first, as the canonical order
// does.
// A type that is not listed falls back to the canonical comparison, which is always correct.
switch (_typed_type->get_primitive_type()) {
// Exact numerics and temporal values: both orders are the numeric/chronological order.
case TYPE_BOOLEAN:
case TYPE_TINYINT:
case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_BIGINT:
case TYPE_DECIMALV2:
case TYPE_DECIMAL32:
case TYPE_DECIMAL64:
case TYPE_DECIMAL128I:
case TYPE_DATE:
case TYPE_DATEV2:
case TYPE_DATETIME:
case TYPE_DATETIMEV2:
case TYPE_TIMESTAMPTZ:
// Strings: both orders compare the bytes as unsigned.
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING:
break;
// Canonical order puts every NaN last and equal to each other, and -0 equal to +0. compare_at()
// of a floating point column orders the same way; its compare_internal() uses < and >, which
// leave NaN unordered.
case TYPE_FLOAT:
case TYPE_DOUBLE:
if (!allow_floating) {
return {nullptr, nullptr};
}
break;
// Not admitted: LARGEINT magnitudes above 10^38 - 1 are Variant strings, and IPv4/IPv6 are
// Variant strings whose byte order is not the address order.
default:
return {nullptr, nullptr};
}
return {&assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(typed_column()),
&assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(right.typed_column())};
}

int ColumnVariantV2::compare_at(size_t n, size_t m, const IColumn& rhs,
int nan_direction_hint) const {
const auto& right = assert_cast<const ColumnVariantV2&, TypeCheckOnRelease::DISABLE>(rhs);
DCHECK_LT(n, size());
DCHECK_LT(m, right.size());

if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, true);
left_nullable != nullptr) {
if (left_nullable->is_null_at(n)) {
return right_nullable->is_null_at(m) ? 0 : -1;
}
if (right_nullable->is_null_at(m)) {
return 1;
}
return left_nullable->get_nested_column().compare_at(
n, m, right_nullable->get_nested_column(), nan_direction_hint);
}

// A column is in exactly one of three states. Encoded rows are canonical Variant bytes that
// get_value_ref() reads in place. A shredded column has no row bytes until its state
// materializes them; get_value_ref() asks the state for them, which builds and caches them on
// the first call. A typed row is a scalar, compared below without being encoded: a sort compares
// O(n log n) times, so nothing here may allocate or encode per comparison.
if (!is_typed() && !right.is_typed()) {
return canonical_compare(get_value_ref(n), right.get_value_ref(m));
}
if (!is_typed()) {
// Keep the typed side on the left; canonical_compare() is antisymmetric.
return -right.compare_at(m, n, *this, nan_direction_hint);
}
// The typed fast path declined (different typed types, a type outside its whitelist, or an
// encoded/shredded right side). A typed null row is the canonical null scalar, which sorts
// first - the same order the typed fast path applies.
int result = 0;
const auto& left_nullable =
assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(typed_column());
visit_typed_scalar_column(
left_nullable, _typed_type->get_primitive_type(), _typed_type->get_scale(), n, n + 1,
[&](size_t, const VariantScalarRef& left_scalar) {
if (!right.is_typed()) {
result = canonical_compare(left_scalar, right.get_value_ref(m));
return;
}
visit_typed_scalar_column(
assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(
right.typed_column()),
right._typed_type->get_primitive_type(), right._typed_type->get_scale(), m,
m + 1, [&](size_t, const VariantScalarRef& right_scalar) {
result = canonical_compare(left_scalar, right_scalar);
});
});
return result;
}

void ColumnVariantV2::compare_internal(size_t rhs_row_id, const IColumn& rhs,
int nan_direction_hint, int direction,
std::vector<uint8_t>& cmp_res,
uint8_t* __restrict filter) const {
const auto& right = assert_cast<const ColumnVariantV2&, TypeCheckOnRelease::DISABLE>(rhs);
// ColumnVector::compare_internal does not provide Variant's canonical NaN ordering, so floating
// point typed columns take the canonical path below.
if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, false);
left_nullable != nullptr) {
left_nullable->compare_internal(rhs_row_id, *right_nullable, -1, direction, cmp_res,
filter);
return;
}
IColumn::compare_internal(rhs_row_id, rhs, nan_direction_hint, direction, cmp_res, filter);
}

Field ColumnVariantV2::operator[](size_t row) const {
Field result;
get(row, result);
Expand Down Expand Up @@ -1662,6 +1816,29 @@ void ColumnVariantV2::pop_back(size_t length) {
_check_invariants();
}

void ColumnVariantV2::erase(size_t start, size_t length) {
DORIS_CHECK_LE(start, size()) << "erase start exceeds the column size";
DORIS_CHECK_LE(length, size() - start) << "erase range exceeds the column size";
if (length == 0) {
return;
}
if (_shredded) {
ensure_encoded();
}
if (_typed) {
mutate_subcolumn(_typed);
_typed->erase(start, length);
_check_invariants();
return;
}
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
// The metadata dictionary keeps blobs that no remaining row references; ids stay valid.
_values->erase(start, length);
_meta_ids->erase(start, length);
_check_invariants();
}

StringRef ColumnVariantV2::get_data_at(size_t) const {
throw_unsupported("get_data_at");
}
Expand Down Expand Up @@ -2139,8 +2316,32 @@ void ColumnVariantV2::resize(size_t new_size) {
}
}

void ColumnVariantV2::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const {
throw_unsupported("get_permutation");
void ColumnVariantV2::get_permutation(bool reverse, size_t limit, int nan_direction_hint,
HybridSorter& sorter, Permutation& result) const {
const size_t row_count = size();
result.resize(row_count);
for (size_t row = 0; row < row_count; ++row) {
result[row] = row;
}

const auto less = [&](size_t left, size_t right) {
const int comparison = compare_at(left, right, *this, nan_direction_hint);
return reverse ? comparison > 0 : comparison < 0;
};
// std::partial_sort only pays off when limit << row_count, the same threshold ColumnVector uses.
if (static_cast<double>(limit) > static_cast<double>(row_count) / 8.0) {
limit = 0;
}
if (limit != 0 && limit < row_count) {
std::partial_sort(result.begin(), result.begin() + limit, result.end(), less);
} else {
sorter.sort(result.begin(), result.end(), less);
}
}

void ColumnVariantV2::sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms,
EqualRange& range, bool last_column) const {
sorter->sort_column(*this, flags, perms, range, last_column);
}

void ColumnVariantV2::replace_column_data(const IColumn&, size_t, size_t) {
Expand Down
27 changes: 27 additions & 0 deletions be/src/core/column/variant_v2/column_variant_v2.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <optional>
#include <span>
#include <string>
#include <utility>

#include "common/exception.h"
#include "core/assert_cast.h"
Expand Down Expand Up @@ -179,7 +180,21 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {

// The input must be an exact, non-Const ColumnNullable whose nested column matches the
// non-nullable supported scalar type.
//
// Every row that is non-null in the typed column's own null map must be a value Variant can
// encode. That null map is the only mask every consumer honours: hashing, for one, maps a row
// to a scalar before it looks at an outer null map. Nothing is checked here, because validity
// is enforced where a row becomes a scalar (with_variant_typed_scalar() throws), which covers
// hashing, serialization, encoding and the canonical comparison. The native comparison paths
// do not go through that mapping and rely on the rule instead:
// - producers that decode stored or transported Variant data build valid rows; an external
// Parquet file is trusted to follow its specification, and a violation surfaces as an
// error when the row is hashed or encoded;
// - CAST from a scalar takes arbitrary values, so it folds its null map into the typed one
// and calls validate_typed_rows().
static MutablePtr create_typed(ColumnPtr column, DataTypePtr scalar_type);
// Throws unless every non-null typed row is a value Variant can encode.
void validate_typed_rows() const;
static MutablePtr create_shredded(std::shared_ptr<VariantShreddedState> state);

bool is_typed() const noexcept { return _typed != nullptr; }
Expand Down Expand Up @@ -224,6 +239,11 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
// including insert, clear, COW mutation, or future row transformations, may invalidate it.
VariantRef get_value_ref(size_t row) const;

int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override;
void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint,
int direction, std::vector<uint8_t>& cmp_res,
uint8_t* __restrict filter) const override;

Field operator[](size_t row) const override;
void get(size_t row, Field& result) const override;
void insert(const Field& field) override;
Expand All @@ -235,6 +255,8 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void pop_back(size_t length) override;
// The analytic sink erases the rows it has consumed from its partition and order key columns.
void erase(size_t start, size_t length) override;

StringRef get_data_at(size_t row) const override;
void insert_data(const char* pos, size_t length) override;
Expand Down Expand Up @@ -272,6 +294,8 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {

void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter,
Permutation& result) const override;
void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms,
EqualRange& range, bool last_column) const override;
void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override;

private:
Expand All @@ -289,6 +313,9 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
void _adopt_state_from(ColumnVariantV2& replacement);
void _detach_metadata_for_write();
void _check_invariants() const;
std::pair<const ColumnNullable*, const ColumnNullable*> _typed_ordering_operands(
const ColumnVariantV2& right, bool allow_floating) const;

static void mutate_subcolumn(IColumn::WrappedPtr& subcolumn) {
static_cast<IColumn::Ptr&>(subcolumn) =
std::move(*static_cast<const IColumn::Ptr&>(subcolumn)).mutate();
Expand Down
8 changes: 4 additions & 4 deletions be/src/core/data_type/data_type_variant_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ MutableColumnPtr DataTypeVariantV2::create_column() const {
}

bool DataTypeVariantV2::equals(const IDataType& rhs) const {
const auto* rhs_type = typeid_cast<const DataTypeVariantV2*>(&rhs);
return rhs_type != nullptr &&
_max_subcolumns_count == rhs_type->variant_max_subcolumns_count() &&
_enable_doc_mode == rhs_type->enable_doc_mode();
// The max subcolumns count and doc mode only guide how storage shreds a Variant column.
// Execution compares, hashes and serializes Variant values canonically without them, so two
// Variant types with different properties are the same execution type (e.g. hash join keys).
return typeid_cast<const DataTypeVariantV2*>(&rhs) != nullptr;
}

int64_t DataTypeVariantV2::get_uncompressed_serialized_bytes(const IColumn& column,
Expand Down
5 changes: 3 additions & 2 deletions be/src/core/data_type_serde/data_type_array_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,8 @@ Status DataTypeArraySerDe::serialize_one_cell_to_hive_text(
}

Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const {
JsonbWriter& writer,
const FormatOptions& options) const {
const auto& data_column = assert_cast<const ColumnArray&>(from_column);
const auto& offsets = data_column.get_offsets();

Expand All @@ -241,7 +242,7 @@ Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column,
}

for (size_t i = start; i < end; ++i) {
RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer));
RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer, options));
}
if (!writer.writeEndArray()) {
return Status::InternalError("writeEndArray failed");
Expand Down
3 changes: 2 additions & 1 deletion be/src/core/data_type_serde/data_type_array_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
const FormatOptions& options) const override;

Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const override;
JsonbWriter& writer,
const FormatOptions& options) const override;

Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const override;
Expand Down
13 changes: 9 additions & 4 deletions be/src/core/data_type_serde/data_type_decimal_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1239,8 +1239,8 @@ void DataTypeDecimalSerDe<T>::to_string_batch(const IColumn& column, ColumnStrin

template <PrimitiveType T>
Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb(const IColumn& from_column,
int64_t row_num,
JsonbWriter& writer) const {
int64_t row_num, JsonbWriter& writer,
const FormatOptions& options) const {
if constexpr (T == TYPE_DECIMALV2) {
return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb");
} else {
Expand All @@ -1254,15 +1254,20 @@ Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb(const IColumn& from_co
}

template <PrimitiveType T>
Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb_vector(const IColumn& from_column,
ColumnString& to_column) const {
Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb_vector(
const IColumn& from_column, ColumnString& to_column, const FormatOptions& options,
const NullMap::value_type* source_null_map) const {
if constexpr (T == TYPE_DECIMALV2) {
return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb_vector");
} else {
const auto size = from_column.size();
JsonbWriter writer;
const auto& data = assert_cast<const ColumnDecimal<T>&>(from_column).get_data();
for (int i = 0; i < size; i++) {
if (source_null_map && source_null_map[i]) {
to_column.insert_default();
continue;
}
writer.reset();
if (!writer.writeDecimal(data[i], precision, scale)) {
return Status::InvalidArgument(
Expand Down
Loading
Loading