Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f326b66
[feature](variant) Add a serialize type so that count 0 keeps every d…
csun5285 Oct 28, 2026
d16534a
[feature](changelog) Support VARIANT columns in changelog
csun5285 Oct 28, 2026
9a2db12
[fix](changelog) Skip the variant path stats check on changelog tablets
csun5285 Oct 29, 2026
58fae14
[test](changelog) Run existing changelog, stream, time travel and IVM…
csun5285 Oct 29, 2026
da3e987
[fix](variant) Check materialized inputs per segment and count BEFORE…
csun5285 Oct 29, 2026
eecfa49
[fix](variant) Keep the fuzzy variant subcolumn count above 0 in doc …
csun5285 Oct 29, 2026
d4c8fc7
[improvement](variant) Check the V2 count-0 layout in the reader and …
csun5285 Oct 30, 2026
ed97a67
[test](regression) Add the serialize type to the Paimon VARIANT DESC …
csun5285 Oct 30, 2026
6074bad
[chore](variant) Shorten the serialize type comments in the reader an…
csun5285 Oct 30, 2026
251d871
[fix](variant) Skip sub-path pruning for changelog() and print old VA…
csun5285 Oct 30, 2026
3a3c185
[improvement](variant) Make the serialize type an internal field
csun5285 Oct 30, 2026
95a4033
[fix](changelog) Reject doc mode sources and keep other scans pruned …
csun5285 Nov 2, 2026
008d696
[improvement](changelog) Keep the statement-wide pruning guard for Va…
csun5285 Nov 2, 2026
fe4b44a
[chore](variant) Merge master into variant-serialize-type
csun5285 Nov 3, 2026
24cb6a2
[test](regression) Pin the subcolumn count of the VARIANT column in t…
csun5285 Nov 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions be/src/cloud/cloud_meta_mgr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1694,8 +1694,6 @@ Status CloudMetaMgr::update_tmp_rowset(const RowsetMeta& rs_meta, int64_t table_
<< ", rowset_id: " << rs_meta.rowset_id()
<< ", attach_changelog_tablet_id: " << attach_changelog->tablet_id()
<< ", attach_changelog_rowset_id: " << attach_changelog->rowset_id();
DCHECK_EQ(rs_meta.tablet_schema()->num_variant_columns(),
attach_changelog->tablet_schema()->num_variant_columns());
RETURN_IF_ERROR(do_update_tmp_rowset(*attach_changelog, table_id));
return do_update_tmp_rowset(rs_meta, table_id);
}
Expand Down
19 changes: 13 additions & 6 deletions be/src/exec/common/variant_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -854,8 +854,14 @@ Status VariantCompactionUtil::aggregate_variant_extended_info(

// get the subpaths and sparse paths for the variant column
void VariantCompactionUtil::get_subpaths(int32_t max_subcolumns_count,
VariantSerializeTypePB serialize_type,
const PathToNoneNullValues& stats,
VariantCompactionPaths& paths_set_info) {
// Count 0 means no limit under V1, and no dynamic subcolumn under V2 (every path stays sparse).
if (max_subcolumns_count == 0 &&
serialize_type == VariantSerializeTypePB::VARIANT_SERIALIZE_V2) {
return;
}
// max_subcolumns_count is 0 means no limit
if (max_subcolumns_count > 0 && stats.size() > max_subcolumns_count) {
std::vector<std::pair<size_t, std::string_view>> paths_with_sizes;
Expand Down Expand Up @@ -919,8 +925,9 @@ Status VariantCompactionUtil::check_path_stats(const std::vector<RowsetSharedPtr
}
}
#endif
// only check path stats for dup_keys since the rows may be merged in other models
if (tablet->keys_type() != KeysType::DUP_KEYS) {
// only check path stats for dup_keys since the rows may be merged in other models; a changelog
// tablet is dup_keys, but its compaction drops the rows its delete bitmap marks
if (tablet->keys_type() != KeysType::DUP_KEYS || tablet->is_changelog_tablet()) {
return Status::OK();
}
// if there is a delete predicate in the input rowsets, we skip the path stats check
Expand Down Expand Up @@ -1233,8 +1240,8 @@ Status VariantCompactionUtil::get_extended_compaction_schema(
output_schema, paths_set_info));

// 3. get the subpaths
get_subpaths(column->variant_max_subcolumns_count(), extended_info.path_to_none_null_values,
paths_set_info);
get_subpaths(column->variant_max_subcolumns_count(), column->variant_serialize_type(),
extended_info.path_to_none_null_values, paths_set_info);

// 4. append subcolumns
if (column->variant_max_subcolumns_count() > 0 || !column->get_sub_columns().empty()) {
Expand All @@ -1243,8 +1250,8 @@ Status VariantCompactionUtil::get_extended_compaction_schema(
extended_info.sparse_paths, output_schema);
}
// variant_max_subcolumns_count == 0 and no typed paths materialized
// it means that all subcolumns are materialized, may be from old data
else {
// under V1 it means that all subcolumns are materialized, may be from old data
else if (column->variant_serialize_type() == VariantSerializeTypePB::VARIANT_SERIALIZE_V1) {
get_compaction_subcolumns_from_data_types(paths_set_info, column, target,
extended_info.path_to_data_types,
output_schema);
Expand Down
3 changes: 2 additions & 1 deletion be/src/exec/common/variant_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ bool generate_sub_column_info(const TabletSchema& schema, int32_t col_unique_id,
class VariantCompactionUtil {
public:
// get the subpaths and sparse paths for the variant column
static void get_subpaths(int32_t max_subcolumns_count, const PathToNoneNullValues& path_stats,
static void get_subpaths(int32_t max_subcolumns_count, VariantSerializeTypePB serialize_type,
const PathToNoneNullValues& path_stats,
VariantCompactionPaths& paths_set_info);

// collect extended info from the variant column
Expand Down
13 changes: 8 additions & 5 deletions be/src/storage/segment/variant/v2/variant_path_builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1399,7 +1399,8 @@ Status VariantPathBuilder::write_sparse_cell(size_t value_index, ColumnString::C

VariantPathSelection select_variant_paths(std::span<const VariantPathSelectionCandidate> candidates,
size_t max_dynamic_materialized_paths,
bool typed_paths_to_sparse) {
bool typed_paths_to_sparse,
VariantSerializeTypePB serialize_type) {
struct DynamicCandidate {
size_t index = 0;
uint32_t non_null_rows = 0;
Expand Down Expand Up @@ -1433,10 +1434,12 @@ VariantPathSelection select_variant_paths(std::span<const VariantPathSelectionCa
return left_path.get_path() > right_path.get_path();
});

const size_t selected_dynamic =
max_dynamic_materialized_paths == 0
? dynamic.size()
: std::min(max_dynamic_materialized_paths, dynamic.size());
size_t selected_dynamic = std::min(max_dynamic_materialized_paths, dynamic.size());
// A count of 0 materializes every dynamic path under V1 and none under V2.
if (max_dynamic_materialized_paths == 0 &&
serialize_type == VariantSerializeTypePB::VARIANT_SERIALIZE_V1) {
selected_dynamic = dynamic.size();
}
for (size_t index = 0; index < dynamic.size(); ++index) {
(index < selected_dynamic ? result.materialized : result.sparse)
.push_back(dynamic[index].index);
Expand Down
5 changes: 4 additions & 1 deletion be/src/storage/segment/variant/v2/variant_path_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

#pragma once

#include <gen_cpp/olap_file.pb.h>

#include <cstddef>
#include <cstdint>
#include <functional>
Expand Down Expand Up @@ -114,7 +116,8 @@ struct VariantPathSelection {
// authoritative non-null-count/depth/path rule before the materialization budget is applied.
VariantPathSelection select_variant_paths(std::span<const VariantPathSelectionCandidate> candidates,
size_t max_dynamic_materialized_paths,
bool typed_paths_to_sparse);
bool typed_paths_to_sparse,
VariantSerializeTypePB serialize_type);

// Least common type for a variant path. Used to merge types inferred by independent value chunks.
DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right);
Expand Down
5 changes: 3 additions & 2 deletions be/src/storage/segment/variant/v2/variant_shredder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,9 @@ struct VariantShredder::Impl {
DorisVector<size_t> all(candidate_builders.size());
std::iota(all.begin(), all.end(), 0);
RETURN_IF_ERROR(convert_typed_candidates(all, candidate_builders, storage_types));
const VariantPathSelection selection = select_variant_paths(
candidates, options.max_subcolumns_count, options.typed_paths_to_sparse);
const VariantPathSelection selection =
select_variant_paths(candidates, options.max_subcolumns_count,
options.typed_paths_to_sparse, options.serialize_type);
publish_root(result);
RETURN_IF_ERROR(publish_materialized(selection, candidate_builders, result));
RETURN_IF_ERROR(publish_sparse(selection, candidate_builders, result));
Expand Down
4 changes: 4 additions & 0 deletions be/src/storage/segment/variant/v2/variant_shredder.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

#pragma once

#include <gen_cpp/olap_file.pb.h>

#include <cstddef>
#include <cstdint>
#include <memory>
Expand Down Expand Up @@ -49,6 +51,8 @@ struct VariantShredderOptions {
// Zero retains all dynamic paths, matching variant_max_subcolumns_count semantics.
size_t max_subcolumns_count = 0;
bool typed_paths_to_sparse = false;
// What max_subcolumns_count == 0 means: every dynamic path (V1) or none (V2).
VariantSerializeTypePB serialize_type = VariantSerializeTypePB::VARIANT_SERIALIZE_V1;
uint32_t sparse_bucket_count = 1;
size_t max_sparse_column_statistics_size =
BeConsts::DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATS_SIZE;
Expand Down
30 changes: 28 additions & 2 deletions be/src/storage/segment/variant/variant_column_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,27 @@ class ReaderOwnedColumnIterator final : public ColumnIterator {
std::shared_ptr<ColumnReader> _owner;
};

// V2 with count 0 keeps dynamic paths (and typed ones with typed_paths_to_sparse) in the sparse
// column, so a template path `t` may be a subcolumn but a dynamic `a` may not.
void check_variant_serialize_type_layout(const TabletColumn& column,
const SubcolumnColumnMetaInfo& subcolumns) {
if (column.variant_serialize_type() != VariantSerializeTypePB::VARIANT_SERIALIZE_V2 ||
column.variant_max_subcolumns_count() != 0 || column.variant_enable_doc_mode() ||
column.variant_enable_nested_group()) {
return;
}
for (const auto& entry : subcolumns) {
if (entry->path.empty()) {
continue; // the root
}
DORIS_CHECK(entry->path.get_is_typed() && !column.variant_enable_typed_paths_to_sparse())

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Accept historical layouts written by old cloud BEs

With the default write_schema_kv=true, upgrade the FE before all BEs, create an ordinary count-zero VARIANT column, and load {"a":1} through an old BE. That BE ignores the new policy field and materializes a. However, CloudInternalCatalog has already persisted the V2 root schema; commit_rowset separates dynamic leaves into the dictionary while put_schema_kv retains that existing root. get_rowset therefore reconstructs a V2/count-zero schema for this materialized segment. A normal read after syncing it onto a fresh/upgraded BE reaches this check and fails (exception in Release, abort in Debug), making committed rows unreadable.

This is distinct from existing compaction finding: it affects ordinary cloud reads without compaction, changelog, or a later schema change. Preserve/recognize the producer's physical layout, or gate V2 creation until writers support it, and cover an old-BE write followed by an upgraded-BE cloud read.

<< "variant column " << column.name()
<< " uses serialize type V2 with max subcolumns count 0, but the segment "
"materializes path "
<< entry->path.get_path();
}
}

} // namespace

const SubcolumnColumnMetaInfo::Node* VariantColumnReader::get_subcolumn_meta_by_path(
Expand Down Expand Up @@ -1398,6 +1419,8 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
num_rows, _nested_group_readers));
}

check_variant_serialize_type_layout(_tablet_schema->column_by_uid(_root_unique_id),
*_subcolumns_meta_info);
return Status::OK();
}

Expand Down Expand Up @@ -1466,8 +1489,11 @@ Status VariantColumnReader::load_external_meta_once(OlapReaderStatistics* stats,
_tablet_schema->column_by_uid(_root_unique_id))
? _statistics.get()
: nullptr;
return _ext_meta_reader->load_all_once(_subcolumns_meta_info.get(), variant_stats, stats,
source_io_ctx);
RETURN_IF_ERROR(_ext_meta_reader->load_all_once(_subcolumns_meta_info.get(), variant_stats,
stats, source_io_ctx));
check_variant_serialize_type_layout(_tablet_schema->column_by_uid(_root_unique_id),
*_subcolumns_meta_info);
return Status::OK();
}

TabletIndexes VariantColumnReader::find_subcolumn_tablet_indexes(const TabletColumn& column,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ Status make_variant_shredder_options(const TabletSchema& tablet_schema,
.physical_layout = physical_layout,
.max_subcolumns_count = cast_set<size_t>(parent_column.variant_max_subcolumns_count()),
.typed_paths_to_sparse = parent_column.variant_enable_typed_paths_to_sparse(),
.serialize_type = parent_column.variant_serialize_type(),
.sparse_bucket_count = cast_set<uint32_t>(
std::max(1, parent_column.variant_sparse_hash_shard_count())),
.max_sparse_column_statistics_size =
Expand Down
10 changes: 10 additions & 0 deletions be/src/storage/tablet/tablet_meta.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,16 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
if (tcolumn.column_type.__isset.variant_enable_doc_mode) {
column->set_variant_enable_doc_mode(tcolumn.column_type.variant_enable_doc_mode);
}
if (tcolumn.__isset.variant_serialize_type) {
switch (tcolumn.variant_serialize_type) {
case TVariantSerializeType::V1:
column->set_variant_serialize_type(VariantSerializeTypePB::VARIANT_SERIALIZE_V1);
break;
case TVariantSerializeType::V2:
column->set_variant_serialize_type(VariantSerializeTypePB::VARIANT_SERIALIZE_V2);
break;
}
}
if (tcolumn.__isset.variant_doc_materialization_min_rows) {
column->set_variant_doc_materialization_min_rows(
tcolumn.variant_doc_materialization_min_rows);
Expand Down
4 changes: 4 additions & 0 deletions be/src/storage/tablet/tablet_schema.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
if (column.has_variant_enable_nested_group()) {
_variant.enable_nested_group = column.variant_enable_nested_group();
}
if (column.has_variant_serialize_type()) {
_variant.serialize_type = column.variant_serialize_type();
}
if (column.has_pattern_type()) {
_pattern_type = column.pattern_type();
}
Expand Down Expand Up @@ -664,6 +667,7 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
column->set_variant_doc_materialization_min_rows(_variant.doc_materialization_min_rows);
column->set_variant_doc_hash_shard_count(_variant.doc_hash_shard_count);
column->set_variant_enable_nested_group(_variant.enable_nested_group);
column->set_variant_serialize_type(_variant.serialize_type);
}

void TabletColumn::add_sub_column(TabletColumn& sub_column) {
Expand Down
6 changes: 6 additions & 0 deletions be/src/storage/tablet/tablet_schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ class TabletColumn : public MetadataAdder<TabletColumn> {
int32_t doc_hash_shard_count = 64;

bool enable_nested_group = false;

VariantSerializeTypePB serialize_type = VariantSerializeTypePB::VARIANT_SERIALIZE_V1;
};

TabletColumn();
Expand Down Expand Up @@ -284,6 +286,10 @@ class TabletColumn : public MetadataAdder<TabletColumn> {

void set_variant_enable_nested_group(bool val) { _variant.enable_nested_group = val; }

VariantSerializeTypePB variant_serialize_type() const { return _variant.serialize_type; }

void set_variant_serialize_type(VariantSerializeTypePB val) { _variant.serialize_type = val; }

bool is_decimal() const { return _is_decimal; }

private:
Expand Down
Loading
Loading