Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
dd35a17
[feature](variant) Support relational operations for Variant
eldenmoon Sep 8, 2026
afd4f61
[improvement](be) Reuse typed Variant comparison and serialization paths
eldenmoon Sep 9, 2026
57d882c
[test](be) Cover typed Variant ordering and nullable serialization
eldenmoon Sep 9, 2026
578d5b6
[test](regression) Cover comparisons between Variant subfields
eldenmoon Sep 9, 2026
810a47a
[test](regression) Extend Variant comparison boundary coverage
eldenmoon Sep 9, 2026
17e33b7
[fix](be) Declare forced-inline canonical hashing helper inline
eldenmoon Sep 10, 2026
f955e5d
[test](regression) Align Variant equality and ordering expectations
eldenmoon Sep 10, 2026
5019ca1
[improvement](be) Fuse Variant nullable serialization traversal
eldenmoon Sep 11, 2026
e0870e0
[test](regression) Align bitmap filter error text
eldenmoon Sep 11, 2026
79b8411
[fix](fe) Adapt Variant comparisons to V2-only master
eldenmoon Sep 15, 2026
1940ca9
[test](regression) Align Variant tests with V2-only master
eldenmoon Sep 15, 2026
f2441d9
[fix](variant) Fix Variant join key types, LARGEINT ordering and inva…
eldenmoon Sep 15, 2026
3978f57
[test](regression) Expect Variant equality joins in test_join_on
eldenmoon Sep 15, 2026
12f8e0b
[fix](nereids) Accept a Variant partition key in window functions
eldenmoon Sep 16, 2026
7e8619d
[improvement](variant) Reuse existing helpers in Variant comparison a…
eldenmoon Sep 16, 2026
f029b3c
[test](variant) Pin how a Variant path that mixes booleans and number…
eldenmoon Sep 16, 2026
b5799e4
[test](variant) Cover the PartitionTopN shape and pin the partial sor…
eldenmoon Sep 16, 2026
780543a
[fix](nereids) Accept a bare NULL in Variant equality comparisons
eldenmoon Sep 17, 2026
b664331
[improvement](variant) Compare typed Variant rows without encoding an…
eldenmoon Sep 17, 2026
b5b4c41
[test](variant) Record how batching, tablets and compaction change a …
eldenmoon Sep 17, 2026
6a20f48
[test](regression) Validate the dimension tables and separate RF, Top…
eldenmoon Sep 17, 2026
f319228
[test](pytest) Match the metric-type error message by its stable prefix
eldenmoon Sep 17, 2026
f572343
[fix](variant) Implement erase for ColumnVariantV2 so window function…
eldenmoon Sep 17, 2026
b7f43d8
[test](variant) Pin a boolean-only Variant path across loads, tablets…
eldenmoon Sep 18, 2026
367727f
[test](regression) Keep the shared metric-type error message unchanged
eldenmoon Sep 18, 2026
5268d61
[refactor](variant) Revert the schema-template auto cast (#60362)
eldenmoon Sep 18, 2026
30096ac
[feature](variant) Let JSON functions accept Variant arguments
eldenmoon Sep 21, 2026
21b1da4
[test](variant) Keep the Variant JSON function case independent of st…
eldenmoon Sep 22, 2026
29d791e
[improvement](variant) Serialize Variant to JSONB in the BE for to_json
eldenmoon Sep 22, 2026
050b83b
Merge apache/doris master into the Variant relational branch
eldenmoon Sep 23, 2026
67d7d84
[fix](fe) Retire enable_variant_schema_auto_cast instead of dropping …
eldenmoon Sep 23, 2026
0f7110c
[improvement](variant) Reject TIMESTAMPTZ paths in VARIANT Schema Tem…
eldenmoon Sep 23, 2026
c2b9395
[chore](regression) Merge master and resolve Variant join test conflict
eldenmoon Oct 8, 2026
f89ab55
Merge branch 'master' into branch-cir-20720
eldenmoon Oct 8, 2026
4ebac66
[chore](be) Merge master and preserve JSONB null and timezone handling
eldenmoon Oct 9, 2026
7d9f593
[test](be) Cover masked Variant JSONB vector serialization
eldenmoon Oct 9, 2026
f5f9642
[fix](fe) Reject Variant runtime filters before pushdown
eldenmoon Oct 10, 2026
ff9e1b5
[test](fe) Cover Variant runtime filter context validation
eldenmoon Oct 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 220 additions & 2 deletions be/src/core/column/variant_v2/column_variant_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include "core/value/variant/variant_canonical.h"
#include "core/value/variant/variant_field.h"
#include "core/value/variant/variant_parquet_encoding.h"
#include "exec/sort/sort_block.h"

namespace doris {
namespace {
Expand Down Expand Up @@ -671,6 +672,34 @@ const IColumn& ColumnVariantV2::typed_column() const {
return *_typed;
}

void ColumnVariantV2::validate_typed_rows() const {
DORIS_CHECK(_typed != nullptr) << "validate_typed_rows requires ColumnVariantV2 typed state";
switch (_typed_type->get_primitive_type()) {
// Every value of these types maps to a Variant scalar, so there is nothing to reject and no
// reason to format each LARGEINT or IP address only to discard the text.
case TYPE_BOOLEAN:
case TYPE_TINYINT:
case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_BIGINT:
case TYPE_LARGEINT:
case TYPE_FLOAT:
case TYPE_DOUBLE:
case TYPE_TIMESTAMP_NS:
case TYPE_IPV4:
case TYPE_IPV6:
return;
default:
break;
}
// with_variant_typed_scalar() is the one definition of what Variant can encode: it throws for
// an invalid date, a string that is not UTF-8 or a decimal outside its width. A type missing
// from the list above is validated, which is the safe side.
visit_typed_scalar_column(assert_cast<const ColumnNullable&>(*_typed),
_typed_type->get_primitive_type(), _typed_type->get_scale(), 0,
_typed->size(), [](size_t, const VariantScalarRef&) {});
}

const DataTypePtr& ColumnVariantV2::typed_type() const {
DORIS_CHECK(_typed_type != nullptr) << "typed_type requires ColumnVariantV2 typed state";
return _typed_type;
Expand Down Expand Up @@ -1037,6 +1066,133 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row) const {
return {.metadata = {.data = metadata.data, .size = metadata.size}, .value = value};
}

std::pair<const ColumnNullable*, const ColumnNullable*> ColumnVariantV2::_typed_ordering_operands(
const ColumnVariantV2& right, bool allow_floating) const {
if (!is_typed() || !right.is_typed() ||
!(_typed_type == right._typed_type || _typed_type->equals(*right._typed_type))) {
return {nullptr, nullptr};
}
// Admission is a whitelist: a typed type takes the native path only after its native order has
// been checked against the canonical Variant order (scalar_compare in variant_canonical.cpp).
// The premises shared by every admitted type:
// - both sides have the same typed type including its parameters, so DECIMAL scales and
// DATETIME precisions match and native values are directly comparable;
// - every non-null typed row is a value Variant can encode (the rule and who enforces it
// are at create_typed()), so the native order does not rank a value the canonical path
// would reject, such as an invalid date or a string that is not UTF-8;
// - a typed NULL is the Variant null, which the callers order first, as the canonical order
// does.
// A type that is not listed falls back to the canonical comparison, which is always correct.
switch (_typed_type->get_primitive_type()) {
// Exact numerics and temporal values: both orders are the numeric/chronological order.
case TYPE_BOOLEAN:
case TYPE_TINYINT:
case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_BIGINT:
case TYPE_DECIMALV2:
case TYPE_DECIMAL32:
case TYPE_DECIMAL64:
case TYPE_DECIMAL128I:
case TYPE_DATE:
case TYPE_DATEV2:
case TYPE_DATETIME:
case TYPE_DATETIMEV2:
case TYPE_TIMESTAMP_NS:
case TYPE_TIMESTAMPTZ:
// Strings: both orders compare the bytes as unsigned.
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING:
break;
// Canonical order puts every NaN last and equal to each other, and -0 equal to +0. compare_at()
// of a floating point column orders the same way; its compare_internal() uses < and >, which
// leave NaN unordered.
case TYPE_FLOAT:
case TYPE_DOUBLE:
if (!allow_floating) {
return {nullptr, nullptr};
}
break;
// Not admitted: LARGEINT magnitudes above 10^38 - 1 are Variant strings, and IPv4/IPv6 are
// Variant strings whose byte order is not the address order.
default:
return {nullptr, nullptr};
}
return {&assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(typed_column()),
&assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(right.typed_column())};
}

int ColumnVariantV2::compare_at(size_t n, size_t m, const IColumn& rhs,
int nan_direction_hint) const {
const auto& right = assert_cast<const ColumnVariantV2&, TypeCheckOnRelease::DISABLE>(rhs);
DCHECK_LT(n, size());
DCHECK_LT(m, right.size());

if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, true);
left_nullable != nullptr) {
if (left_nullable->is_null_at(n)) {
return right_nullable->is_null_at(m) ? 0 : -1;
}
if (right_nullable->is_null_at(m)) {
return 1;
}
return left_nullable->get_nested_column().compare_at(
n, m, right_nullable->get_nested_column(), nan_direction_hint);
}

// A column is in exactly one of three states. Encoded rows are canonical Variant bytes that
// get_value_ref() reads in place. A shredded column has no row bytes until its state
// materializes them; get_value_ref() asks the state for them, which builds and caches them on
// the first call. A typed row is a scalar, compared below without being encoded: a sort compares
// O(n log n) times, so nothing here may allocate or encode per comparison.
if (!is_typed() && !right.is_typed()) {
return canonical_compare(get_value_ref(n), right.get_value_ref(m));
}
if (!is_typed()) {
// Keep the typed side on the left; canonical_compare() is antisymmetric.
return -right.compare_at(m, n, *this, nan_direction_hint);
}
// The typed fast path declined (different typed types, a type outside its whitelist, or an
// encoded/shredded right side). A typed null row is the canonical null scalar, which sorts
// first - the same order the typed fast path applies.
int result = 0;
const auto& left_nullable =
assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(typed_column());
visit_typed_scalar_column(
left_nullable, _typed_type->get_primitive_type(), _typed_type->get_scale(), n, n + 1,
[&](size_t, const VariantScalarRef& left_scalar) {
if (!right.is_typed()) {
result = canonical_compare(left_scalar, right.get_value_ref(m));
return;
}
visit_typed_scalar_column(
assert_cast<const ColumnNullable&, TypeCheckOnRelease::DISABLE>(
right.typed_column()),
right._typed_type->get_primitive_type(), right._typed_type->get_scale(), m,
m + 1, [&](size_t, const VariantScalarRef& right_scalar) {
result = canonical_compare(left_scalar, right_scalar);
});
});
return result;
}

void ColumnVariantV2::compare_internal(size_t rhs_row_id, const IColumn& rhs,
int nan_direction_hint, int direction,
std::vector<uint8_t>& cmp_res,
uint8_t* __restrict filter) const {
const auto& right = assert_cast<const ColumnVariantV2&, TypeCheckOnRelease::DISABLE>(rhs);
// ColumnVector::compare_internal does not provide Variant's canonical NaN ordering, so floating
// point typed columns take the canonical path below.
if (const auto [left_nullable, right_nullable] = _typed_ordering_operands(right, false);
left_nullable != nullptr) {
left_nullable->compare_internal(rhs_row_id, *right_nullable, -1, direction, cmp_res,
filter);
return;
}
IColumn::compare_internal(rhs_row_id, rhs, nan_direction_hint, direction, cmp_res, filter);
}

Field ColumnVariantV2::operator[](size_t row) const {
Field result;
get(row, result);
Expand Down Expand Up @@ -1465,6 +1621,29 @@ void ColumnVariantV2::pop_back(size_t length) {
_check_invariants();
}

void ColumnVariantV2::erase(size_t start, size_t length) {
DORIS_CHECK_LE(start, size()) << "erase start exceeds the column size";
DORIS_CHECK_LE(length, size() - start) << "erase range exceeds the column size";
if (length == 0) {
return;
}
if (_shredded) {
ensure_encoded();
}
if (_typed) {
mutate_subcolumn(_typed);
_typed->erase(start, length);
_check_invariants();
return;
}
Comment thread
eldenmoon marked this conversation as resolved.
Resolved
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
// The metadata dictionary keeps blobs that no remaining row references; ids stay valid.
_values->erase(start, length);
_meta_ids->erase(start, length);
_check_invariants();
}

StringRef ColumnVariantV2::get_data_at(size_t) const {
throw_unsupported("get_data_at");
}
Expand Down Expand Up @@ -1602,6 +1781,21 @@ void ColumnVariantV2::serialize(StringRef* keys, size_t num_rows) const {
}
}

void ColumnVariantV2::serialize_with_nullable(StringRef* keys, size_t num_rows, bool has_null,
const uint8_t* __restrict null_map) const {
if (has_null) {
IColumn::serialize_with_nullable(keys, num_rows, has_null, null_map);
return;
}
// Without nulls every row carries the same zero flag, so write the flags first and let
// serialize() append each value at the advanced cursor.
for (size_t row = 0; row < num_rows; ++row) {
*(const_cast<char*>(keys[row].data) + keys[row].size) = 0;
keys[row].size += sizeof(UInt8);
}
serialize(keys, num_rows);
}

void ColumnVariantV2::deserialize(StringRef* keys, size_t num_rows) {
DCHECK(keys != nullptr || num_rows == 0);
for (size_t row = 0; row < num_rows; ++row) {
Expand Down Expand Up @@ -1945,8 +2139,32 @@ void ColumnVariantV2::resize(size_t new_size) {
}
}

void ColumnVariantV2::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const {
throw_unsupported("get_permutation");
void ColumnVariantV2::get_permutation(bool reverse, size_t limit, int nan_direction_hint,
HybridSorter& sorter, Permutation& result) const {
const size_t row_count = size();
result.resize(row_count);
for (size_t row = 0; row < row_count; ++row) {
result[row] = row;
}

const auto less = [&](size_t left, size_t right) {
const int comparison = compare_at(left, right, *this, nan_direction_hint);
return reverse ? comparison > 0 : comparison < 0;
};
// std::partial_sort only pays off when limit << row_count, the same threshold ColumnVector uses.
if (static_cast<double>(limit) > static_cast<double>(row_count) / 8.0) {
limit = 0;
}
if (limit != 0 && limit < row_count) {
std::partial_sort(result.begin(), result.begin() + limit, result.end(), less);
} else {
sorter.sort(result.begin(), result.end(), less);
}
}

void ColumnVariantV2::sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms,
EqualRange& range, bool last_column) const {
sorter->sort_column(*this, flags, perms, range, last_column);
}

void ColumnVariantV2::replace_column_data(const IColumn&, size_t, size_t) {
Expand Down
31 changes: 31 additions & 0 deletions be/src/core/column/variant_v2/column_variant_v2.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <optional>
#include <span>
#include <string>
#include <utility>

#include "common/exception.h"
#include "core/assert_cast.h"
Expand Down Expand Up @@ -157,7 +158,21 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {

// The input must be an exact, non-Const ColumnNullable whose nested column matches the
// non-nullable supported scalar type.
//
// Every row that is non-null in the typed column's own null map must be a value Variant can
// encode. That null map is the only mask every consumer honours: hashing, for one, maps a row
// to a scalar before it looks at an outer null map. Nothing is checked here, because validity
// is enforced where a row becomes a scalar (with_variant_typed_scalar() throws), which covers
// hashing, serialization, encoding and the canonical comparison. The native comparison paths
// do not go through that mapping and rely on the rule instead:
// - producers that decode stored or transported Variant data build valid rows; an external
// Parquet file is trusted to follow its specification, and a violation surfaces as an
// error when the row is hashed or encoded;
// - CAST from a scalar takes arbitrary values, so it folds its null map into the typed one
// and calls validate_typed_rows().
static MutablePtr create_typed(ColumnPtr column, DataTypePtr scalar_type);
// Throws unless every non-null typed row is a value Variant can encode.
void validate_typed_rows() const;
static MutablePtr create_shredded(std::shared_ptr<VariantShreddedState> state);

bool is_typed() const noexcept { return _typed != nullptr; }
Expand Down Expand Up @@ -198,6 +213,11 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
// including insert, clear, COW mutation, or future row transformations, may invalidate it.
VariantRef get_value_ref(size_t row) const;

int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override;
void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint,
int direction, std::vector<uint8_t>& cmp_res,
uint8_t* __restrict filter) const override;

Field operator[](size_t row) const override;
void get(size_t row, Field& result) const override;
void insert(const Field& field) override;
Expand All @@ -210,6 +230,8 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void pop_back(size_t length) override;
// The analytic sink erases the rows it has consumed from its partition and order key columns.
void erase(size_t start, size_t length) override;

StringRef get_data_at(size_t row) const override;
void insert_data(const char* pos, size_t length) override;
Expand All @@ -221,6 +243,8 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
size_t deserialize_impl(const char* pos) override;
size_t get_max_row_byte_size() const override;
void serialize(StringRef* keys, size_t num_rows) const override;
void serialize_with_nullable(StringRef* keys, size_t num_rows, bool has_null,
const uint8_t* __restrict null_map) const override;
void deserialize(StringRef* keys, size_t num_rows) override;

void update_hash_with_value(size_t row, SipHash& hash) const override;
Expand All @@ -247,6 +271,8 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {

void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter,
Permutation& result) const override;
void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& perms,
EqualRange& range, bool last_column) const override;
void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override;

private:
Expand All @@ -262,6 +288,11 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
void _adopt_state_from(ColumnVariantV2& replacement);
void _detach_metadata_for_write();
void _check_invariants() const;
// Typed columns of both sides when their shared typed type orders natively the same way the
// canonical Variant order does, {nullptr, nullptr} otherwise. allow_floating is false for
// callers that cannot apply Variant's canonical NaN ordering.
std::pair<const ColumnNullable*, const ColumnNullable*> _typed_ordering_operands(
const ColumnVariantV2& right, bool allow_floating) const;
void mutate_subcolumns() override;

// Encoded state: each row owns a value and references one deduplicated metadata blob. The
Expand Down
8 changes: 4 additions & 4 deletions be/src/core/data_type/data_type_variant_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ MutableColumnPtr DataTypeVariantV2::create_column() const {
}

bool DataTypeVariantV2::equals(const IDataType& rhs) const {
const auto* rhs_type = typeid_cast<const DataTypeVariantV2*>(&rhs);
return rhs_type != nullptr &&
_max_subcolumns_count == rhs_type->variant_max_subcolumns_count() &&
_enable_doc_mode == rhs_type->enable_doc_mode();
// The max subcolumns count and doc mode only guide how storage shreds a Variant column.
// Execution compares, hashes and serializes Variant values canonically without them, so two
// Variant types with different properties are the same execution type (e.g. hash join keys).
return typeid_cast<const DataTypeVariantV2*>(&rhs) != nullptr;
}

int64_t DataTypeVariantV2::get_uncompressed_serialized_bytes(const IColumn& column,
Expand Down
5 changes: 3 additions & 2 deletions be/src/core/data_type_serde/data_type_array_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ Status DataTypeArraySerDe::serialize_one_cell_to_hive_text(
}

Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const {
JsonbWriter& writer,
const FormatOptions& options) const {
const auto& data_column = assert_cast<const ColumnArray&>(from_column);
const auto& offsets = data_column.get_offsets();

Expand All @@ -242,7 +243,7 @@ Status DataTypeArraySerDe::serialize_column_to_jsonb(const IColumn& from_column,
}

for (size_t i = start; i < end; ++i) {
RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer));
RETURN_IF_ERROR(nested_serde->serialize_column_to_jsonb(nested_column, i, writer, options));
}
if (!writer.writeEndArray()) {
return Status::InternalError("writeEndArray failed");
Expand Down
3 changes: 2 additions & 1 deletion be/src/core/data_type_serde/data_type_array_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
const FormatOptions& options) const override;

Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const override;
JsonbWriter& writer,
const FormatOptions& options) const override;

Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const override;
Expand Down
Loading
Loading