From a400b5c67fe7450733752b7b7c5cc6a592fbb489 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sat, 17 Oct 2026 23:39:13 +0800 Subject: [PATCH 1/7] [fix](variant) Stream the VARIANT root column instead of buffering a segment (CIR-22037) ColumnStr rejects more than 4 GiB of uncompressed chars (E-3113). The VARIANT V2 writer kept the whole output segment's root JSONB in one ColumnString before finalize, so a vertical compaction whose segment carried more than 4 GiB of uncompressed variant data failed deterministically. The compressed-byte segment budget cannot prevent this for high-compression-ratio data. Stream the root column: create the root column writer at init and append every input batch to it, so the writer never holds a segment-sized ColumnString. Keep a data-driven segment budget as a guard for the remaining variant buffers: sum the raw (pre-encoding) data bytes of the input segments' VARIANT column metas and clamp the row budget with variant_compaction_max_segment_uncompressed_bytes (2 GiB by default). Segments written before variant writers reported data bytes keep the compressed-size budget. --- .../variant/v2/variant_column_writer.cpp | 41 ++++++++++++++----- .../variant/v2/variant_column_writer.h | 9 ++-- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.cpp b/be/src/storage/segment/variant/v2/variant_column_writer.cpp index 0790f1c8db0fc1..3b57b37fb2cd2d 100644 --- a/be/src/storage/segment/variant/v2/variant_column_writer.cpp +++ b/be/src/storage/segment/variant/v2/variant_column_writer.cpp @@ -109,7 +109,12 @@ Status VariantV2ColumnWriter::init() { _root_only = variant_writer_helpers::has_extracted_variant_columns( *_opts.rowset_ctx->tablet_schema, _tablet_column->unique_id()); if (_root_only) { - _root_jsonb = ColumnString::create(); + // Stream the root column instead of buffering a whole segment: the JSONB ColumnString + // would otherwise have to hold every row of the segment and could exceed the + // ColumnStr 4 GiB limit before finalize. + _root_writer = std::make_unique( + _opts, std::make_shared(*_tablet_column), _opts.file_writer); + RETURN_IF_ERROR(_root_writer->init()); return Status::OK(); } @@ -129,15 +134,14 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num return Status::InternalError("Cannot append ColumnVariantV2 after writer finalization"); } if (_root_only) { - DORIS_CHECK(_root_jsonb); + DORIS_CHECK(_root_writer); if (num_rows > std::numeric_limits::max() - _num_rows) { return Status::InvalidArgument("Variant writer row count overflows size_t"); } ColumnString::MutablePtr batch_root_jsonb; RETURN_IF_ERROR(build_root_only_batch(column, num_rows, outer_nulls, &batch_root_jsonb)); DORIS_CHECK_EQ(batch_root_jsonb->size(), num_rows); - RETURN_IF_CATCH_EXCEPTION( - { _root_jsonb->insert_range_from(*batch_root_jsonb, 0, num_rows); }); + RETURN_IF_ERROR(_append_root_batch(*batch_root_jsonb, outer_nulls)); } else { RETURN_IF_ERROR(variant_writer_helpers::append_variant_v2_to_shredder( _shredder.get(), column, num_rows, outer_nulls)); @@ -152,6 +156,25 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num return Status::OK(); } +Status VariantV2ColumnWriter::_append_root_batch(const ColumnString& batch, + std::span outer_nulls) { + DORIS_CHECK(_root_writer != nullptr); + if (batch.size() == 0) { + return Status::OK(); + } + if (!outer_nulls.empty() && outer_nulls.size() != batch.size()) { + return Status::InvalidArgument("Variant root outer-null span has {} rows, expected {}", + outer_nulls.size(), batch.size()); + } + DorisVector slices(batch.size()); + for (size_t row = 0; row < batch.size(); ++row) { + slices[row] = batch.get_data_at(row).to_slice(); + } + const uint8_t* null_map = + _tablet_column->is_nullable() && !outer_nulls.empty() ? outer_nulls.data() : nullptr; + return _root_writer->append(null_map, slices.data(), batch.size()); +} + Status VariantV2ColumnWriter::_write_root(const IColumn* root_jsonb, int& column_id) { if (!root_jsonb || root_jsonb->size() != _num_rows) { return Status::InvalidArgument("Variant root column has {} rows, expected {}", @@ -231,10 +254,8 @@ Status VariantV2ColumnWriter::finalize() { } if (_root_only) { DORIS_CHECK(_shredder == nullptr); - DORIS_CHECK(_root_jsonb); - int column_id = 0; - RETURN_IF_ERROR(_write_root(_root_jsonb.get(), column_id)); - _root_jsonb.reset(); + DORIS_CHECK(_root_writer != nullptr); + _opts.meta->set_num_rows(_num_rows); _is_finalized = true; return Status::OK(); } @@ -343,8 +364,8 @@ Status VariantV2ColumnWriter::write_bloom_filter_index() { uint64_t VariantV2ColumnWriter::estimate_buffer_size() { if (!_is_finalized) { if (_root_only) { - DORIS_CHECK(_root_jsonb); - return _outer_nulls->byte_size() + _root_jsonb->byte_size(); + DORIS_CHECK(_root_writer != nullptr); + return _outer_nulls->byte_size() + _root_writer->estimate_buffer_size(); } DORIS_CHECK(_shredder != nullptr); return _outer_nulls->byte_size() + _shredder->byte_size(); diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.h b/be/src/storage/segment/variant/v2/variant_column_writer.h index ea982f58a8706c..491cdf5a964d26 100644 --- a/be/src/storage/segment/variant/v2/variant_column_writer.h +++ b/be/src/storage/segment/variant/v2/variant_column_writer.h @@ -38,9 +38,9 @@ class VariantBinaryWriter; class VariantShredder; struct VariantShreddedColumns; -// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append; -// a parent with extracted children retains only its root JSONB column. The writer never retains a -// segment-sized copy of the input execution column. +// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append. +// A parent with extracted children streams its root JSONB column into the root column writer batch +// by batch. The writer never retains a segment-sized copy of the input execution column. class VariantV2ColumnWriter { public: VariantV2ColumnWriter(ColumnWriterOptions opts, const TabletColumn* column); @@ -65,6 +65,8 @@ class VariantV2ColumnWriter { private: Status _write_root(const IColumn* root_jsonb, int& column_id); + // Streams one batch of the root-only JSONB column into the root column writer. + Status _append_root_batch(const ColumnString& batch, std::span outer_nulls); Status _write_materialized(const VariantShreddedColumns& shredded, OlapBlockDataConvertor* converter, int& column_id); Status _write_binary(const VariantShreddedColumns& shredded, OlapBlockDataConvertor* converter, @@ -74,7 +76,6 @@ class VariantV2ColumnWriter { ColumnWriterOptions _opts; const TabletColumn* _tablet_column = nullptr; std::unique_ptr _shredder; - ColumnString::MutablePtr _root_jsonb; ColumnUInt8::MutablePtr _outer_nulls; size_t _num_rows = 0; bool _root_only = false; From 7b0958da7a8d5a65bf633af95d2b17fdcc010077 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sat, 17 Oct 2026 23:50:09 +0800 Subject: [PATCH 2/7] [test](variant) Cover streamed root writes and variant data page bytes (CIR-22037) - Assert the root-only writer keeps its buffer close to one append batch instead of retaining the whole segment. - Assert variant writers report the data page bytes they wrote (writer counters equal the bytes appended to the file right after write_data). --- .../variant_column_writer_reader_test.cpp | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 0e907d089241cb..bedb522ec76d29 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -2375,6 +2375,103 @@ TEST_F(VariantColumnWriterReaderTest, test_write_column_variant_v2_accounts_root EXPECT_EQ(actual[1], "1"); } +TEST_F(VariantColumnWriterReaderTest, test_variant_root_writer_streams_per_batch) { + // A variant with an extracted subcolumn takes the root-only writer path. Root scalars all + // land in the root JSONB column, so the writer must flush them batch by batch instead of + // retaining the whole segment: buffering SegmentRows * value bytes is what used to overflow + // ColumnStr at 4 GiB. + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb.add_column(), 1, "VARIANT", "v", 10, false, false, 0, false, 0, 0, + false); + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + const TabletColumn& parent_column = _tablet_schema->column_by_uid(1); + TabletColumn extracted_column; + extracted_column.set_name(parent_column.name_lower_case() + ".payload"); + extracted_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + extracted_column.set_parent_unique_id(parent_column.unique_id()); + extracted_column.set_path_info(PathInData(parent_column.name_lower_case() + ".payload")); + extracted_column.set_is_nullable(true); + _tablet_schema->append_column(extracted_column); + init_tablet_from_current_schema(10043); + + constexpr size_t kRows = 4000; + constexpr size_t kBatchRows = 1000; + constexpr size_t kValueBytes = 4096; + const std::string payload(kValueBytes, 'x'); + auto source = ColumnVariantV2::create(); + DataTypeVariantV2SerDe serde; + DataTypeSerDe::FormatOptions serde_options; + for (size_t row = 0; row < kRows; ++row) { + const std::string json = "\"" + payload + "\""; + Slice slice(json.data(), json.size()); + ASSERT_TRUE(serde.deserialize_one_cell_from_json(*source, slice, serde_options).ok()); + } + + const std::string rowset_id = "v2_root_streaming"; + SegmentFooterPB footer; + std::string file_path = local_segment_path(_tablet->tablet_path(), rowset_id, 0); + ASSERT_TRUE(io::global_local_filesystem()->delete_file(file_path).ok()); + io::FileWriterPtr file_writer; + ASSERT_TRUE(io::global_local_filesystem()->create_file(file_path, &file_writer).ok()); + + RowsetWriterContext rowset_ctx; + rowset_ctx.write_type = DataWriteType::TYPE_DIRECT; + rowset_ctx.tablet_schema = _tablet_schema; + rowset_ctx.tablet = _tablet; + rowset_ctx.tablet_path = _tablet->tablet_path(); + + TabletColumn column = _tablet_schema->column(0); + ColumnWriterOptions opts; + opts.meta = footer.add_columns(); + opts.compression_type = CompressionTypePB::LZ4; + opts.file_writer = file_writer.get(); + opts.footer = &footer; + opts.rowset_ctx = &rowset_ctx; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + _init_column_meta(opts.meta, 0, column, opts); + + std::unique_ptr writer; + ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(), &writer).ok()); + ASSERT_TRUE(writer->init().ok()); + + OlapBlockDataConvertor converter; + converter.add_column_data_convertor(column); + const auto source_type = std::make_shared(2, false); + uint64_t max_buffered_bytes = 0; + for (size_t row_pos = 0; row_pos < kRows; row_pos += kBatchRows) { + ASSERT_TRUE(converter + .set_source_content_with_specifid_column( + {source->get_ptr(), source_type, column.name()}, row_pos, + kBatchRows, 0) + .ok()); + auto [convert_status, accessor] = converter.convert_column_data(0); + ASSERT_TRUE(convert_status.ok()); + ASSERT_TRUE(writer->append(accessor->get_nullmap(), accessor->get_data(), kBatchRows).ok()); + converter.clear_source_content(0); + max_buffered_bytes = std::max(max_buffered_bytes, writer->estimate_buffer_size()); + } + // The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB); the + // streaming writer holds at most the current page plus the outer-null map. + EXPECT_LT(max_buffered_bytes, 2 * kBatchRows * kValueBytes); + + ASSERT_TRUE(writer->finish().ok()); + ASSERT_TRUE(writer->write_data().ok()); + ASSERT_TRUE(writer->write_ordinal_index().ok()); + ASSERT_TRUE(writer->write_zone_map().ok()); + ASSERT_TRUE(file_writer->close().ok()); + footer.set_num_rows(kRows); + + std::vector> actual; + ASSERT_TRUE(read_variant_root_rows(footer, file_path, &actual).ok()); + ASSERT_EQ(actual.size(), kRows); + for (const auto& row : actual) { + ASSERT_TRUE(row.has_value()); + EXPECT_EQ(row->size(), kValueBytes + 2); + } +} + TEST_F(VariantColumnWriterReaderTest, test_write_column_variant_v2_typed_state) { init_variant_tablet(10009, 1); From 92db91d7b9f1d874e05e07a625621c6427e9160b Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sun, 18 Oct 2026 01:33:55 +0800 Subject: [PATCH 3/7] [fix](variant) Chunk VariantSubcolumnWriter value buffering (CIR-22037) VariantSubcolumnWriter accumulated every value of an output segment in one VariantPathBuilder column, and materialized a second segment-sized dense column in finalize(). A VARIANT-typed extracted subcolumn whose non-null payload exceeds 4 GiB therefore always failed compaction with E-3113. Accumulate values into bounded chunks (256 MiB / 4M values) instead. Each chunk keeps the inferred type; finalize() merges the chunk types with variant_path_least_common_type(), converts every chunk to the final storage type and flushes it to the underlying writer independently, so neither the builder nor the dense materialization ever holds a segment-sized ColumnString. --- be/src/storage/segment/column_writer.h | 13 ++ .../variant/v2/variant_path_builder.cpp | 4 + .../segment/variant/v2/variant_path_builder.h | 3 + .../variant/variant_column_writer_impl.cpp | 135 +++++++++++++----- 4 files changed, 116 insertions(+), 39 deletions(-) diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index d79b35d16479d3..6e5dd9afad5c64 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -651,12 +651,25 @@ class VariantSubcolumnWriter : public ColumnWriter { std::span outer_nulls); Status _ensure_input_format(const VariantColumnData& column); Status _initialize_v2_builder(); + Status _seal_chunk(); + // Bounded value chunks; each chunk is finalized and flushed independently so the writer never + // holds a segment-sized ColumnString. + struct ValueChunk { + std::unique_ptr builder; + size_t row_begin = 0; + size_t rows = 0; + }; + static constexpr size_t kChunkBytes = 256ULL << 20; + static constexpr size_t kChunkValues = 4ULL << 20; bool is_finalized() const; bool _is_finalized = false; ordinal_t _next_rowid = 0; size_t none_null_size = 0; VariantWriterInputFormat _input_format = VariantWriterInputFormat::UNSET; std::unique_ptr _v2_builder; + std::vector _chunks; + size_t _chunk_row_begin = 0; + size_t _chunk_rows = 0; size_t _num_rows = 0; ColumnWriterOptions _opts; std::unique_ptr _writer; diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index 5f6f6990f33496..42083f4ead4ba8 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -1038,6 +1038,10 @@ bool cast_introduced_null(const IColumn& source, const IColumn& target) { } // namespace +DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right) { + return path_least_common_type(left, right); +} + DataTypePtr normalize_variant_path_integer_widths(const DataTypePtr& type) { const DataTypePtr base = remove_nullable(type); if (const auto* array = typeid_cast(base.get())) { diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.h b/be/src/storage/segment/variant/v2/variant_path_builder.h index 7b467a17ae7274..b48ab5f2ac55d4 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.h +++ b/be/src/storage/segment/variant/v2/variant_path_builder.h @@ -107,4 +107,7 @@ VariantPathSelection select_variant_paths(std::span outer_nulls) { - DORIS_CHECK(_v2_builder != nullptr); DORIS_CHECK(column.column_data != nullptr); const auto* source = check_and_get_column(*column.column_data); DORIS_CHECK(source != nullptr); @@ -596,6 +595,11 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ column.row_pos, column.row_pos + num_rows, source->size()); } DORIS_CHECK(outer_nulls.empty() || outer_nulls.size() == num_rows); + if (_v2_builder == nullptr) { + RETURN_IF_ERROR(_initialize_v2_builder()); + _chunk_row_begin = _num_rows; + _chunk_rows = 0; + } const auto append_encoded = [&](const ColumnVariantV2::ReadView& view, size_t begin) -> Status { DORIS_CHECK(!view.is_typed()); @@ -606,7 +610,7 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ const size_t input_row = begin + offset; const VariantRef value = view.value_at(input_row); if (!value.is_null()) { - RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset)); + RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin)); } } return Status::OK(); @@ -614,15 +618,32 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ const auto view = source->read_view(); if (!view.is_typed()) { - return append_encoded(view, column.row_pos); + RETURN_IF_ERROR(append_encoded(view, column.row_pos)); + } else { + auto encoded_batch = ColumnVariantV2::create(); + RETURN_IF_CATCH_EXCEPTION({ + encoded_batch->insert_range_from(*source, column.row_pos, num_rows); + encoded_batch->ensure_encoded(); + }); + RETURN_IF_ERROR(append_encoded(encoded_batch->read_view(), 0)); + } + _chunk_rows = _num_rows + num_rows - _chunk_row_begin; + if (_v2_builder->byte_size() >= kChunkBytes || _v2_builder->non_null_rows() >= kChunkValues) { + RETURN_IF_ERROR(_seal_chunk()); } + return Status::OK(); +} - auto encoded_batch = ColumnVariantV2::create(); - RETURN_IF_CATCH_EXCEPTION({ - encoded_batch->insert_range_from(*source, column.row_pos, num_rows); - encoded_batch->ensure_encoded(); - }); - return append_encoded(encoded_batch->read_view(), 0); +Status VariantSubcolumnWriter::_seal_chunk() { + if (_v2_builder == nullptr) { + return Status::OK(); + } + RETURN_IF_ERROR(_v2_builder->complete_rows(_chunk_rows)); + _chunks.push_back(ValueChunk { + .builder = std::move(_v2_builder), .row_begin = _chunk_row_begin, .rows = _chunk_rows}); + _v2_builder.reset(); + _chunk_rows = 0; + return Status::OK(); } Status VariantSubcolumnWriter::_append(const uint8_t* null_map, const uint8_t** ptr, @@ -652,7 +673,11 @@ uint64_t VariantSubcolumnWriter::estimate_buffer_size() { if (is_finalized()) { return _writer ? _writer->estimate_buffer_size() : 0; } - return _v2_builder ? _v2_builder->byte_size() : 0; + size_t bytes = _v2_builder ? _v2_builder->byte_size() : 0; + for (const auto& chunk : _chunks) { + bytes += chunk.builder->byte_size(); + } + return bytes; } bool VariantSubcolumnWriter::is_finalized() const { @@ -665,51 +690,63 @@ Status VariantSubcolumnWriter::finalize() { } if (_input_format == VariantWriterInputFormat::UNSET) { RETURN_IF_ERROR(_initialize_v2_builder()); + _chunk_row_begin = 0; + _chunk_rows = 0; _input_format = VariantWriterInputFormat::V2; } + RETURN_IF_ERROR(_seal_chunk()); + const auto& parent_column = _opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id()); DORIS_CHECK(!parent_column.variant_enable_nested_group()); - DataTypePtr flush_type; - ColumnPtr flush_values; - DorisVector flush_rowids; - int64_t non_null_value_size = 0; - DORIS_CHECK(_v2_builder != nullptr); - RETURN_IF_ERROR(_v2_builder->complete_rows(_num_rows)); DataTypePtr storage_type; TabletSchema::SubColumnInfo subcolumn_info; + const PathInData relative_path = get_column()->path_info_ptr()->copy_pop_front(); if (variant_util::generate_sub_column_info(*_opts.rowset_ctx->tablet_schema, - parent_column.unique_id(), - _v2_builder->path().get_path(), &subcolumn_info)) { + parent_column.unique_id(), relative_path.get_path(), + &subcolumn_info)) { storage_type = DataTypeFactory::instance().create_data_type(subcolumn_info.column); } - bool publish_builder = _v2_builder->non_null_rows() != 0; + // Merge the types inferred by independent chunks; the result is what a single builder would + // have inferred for the whole segment. + DataTypePtr merged_type; + bool has_values = false; + for (auto& chunk : _chunks) { + if (chunk.builder->non_null_rows() == 0) { + continue; + } + has_values = true; + RETURN_IF_ERROR(chunk.builder->convert_to( + normalize_variant_path_integer_widths(chunk.builder->type()))); + merged_type = merged_type == nullptr + ? chunk.builder->type() + : variant_path_least_common_type(merged_type, chunk.builder->type()); + } + + bool publish_builder = has_values; if (publish_builder) { - RETURN_IF_ERROR(_v2_builder->convert_to( - normalize_variant_path_integer_widths(_v2_builder->type()))); publish_builder = - !variant_path_type_contains_nothing(_v2_builder->type()) || + !variant_path_type_contains_nothing(merged_type) || (storage_type != nullptr && !variant_path_type_contains_nothing(storage_type)); } - if (publish_builder && storage_type != nullptr) { - RETURN_IF_ERROR(_v2_builder->convert_to(storage_type)); - } + DataTypePtr flush_type; + size_t non_null_value_size = 0; if (!publish_builder) { flush_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_TINYINT, true /* is_nullable */); - MutableColumnPtr defaults = flush_type->create_column(); - flush_values = std::move(defaults); } else { - flush_type = _v2_builder->type(); - flush_values = _v2_builder->column(); - const std::span rowids = _v2_builder->rowids(); - flush_rowids.assign(rowids.begin(), rowids.end()); - non_null_value_size = cast_set(flush_rowids.size()); + flush_type = storage_type != nullptr ? storage_type : merged_type; + for (auto& chunk : _chunks) { + if (chunk.builder->non_null_rows() == 0) { + continue; + } + RETURN_IF_ERROR(chunk.builder->convert_to(flush_type)); + non_null_value_size += chunk.builder->rowids().size(); + } } - _v2_builder.reset(); TabletColumn flush_column = variant_util::get_column_by_type( flush_type, get_column()->name(), @@ -727,13 +764,33 @@ Status VariantSubcolumnWriter::finalize() { _indexes, &opts, non_null_value_size, need_record_none_null_value_size)); _opts = opts; - OlapBlockDataConvertor converter; - RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( - flush_column, _writer.get(), &converter, 0, flush_type, flush_values, flush_rowids, - _num_rows)); + if (!publish_builder) { + OlapBlockDataConvertor converter; + MutableColumnPtr defaults = flush_type->create_column(); + RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( + flush_column, _writer.get(), &converter, 0, flush_type, std::move(defaults), {}, + _num_rows)); + } else { + for (auto& chunk : _chunks) { + // Each chunk owns one contiguous row range, so flushing it needs its own converter + // slot. Reuse is not possible: append_sparse_converted_column registers a convertor + // per call. + OlapBlockDataConvertor converter; + if (chunk.builder->non_null_rows() == 0) { + MutableColumnPtr empty = flush_type->create_column(); + RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( + flush_column, _writer.get(), &converter, 0, flush_type, std::move(empty), + {}, chunk.rows)); + continue; + } + RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( + flush_column, _writer.get(), &converter, 0, flush_type, chunk.builder->column(), + chunk.builder->rowids(), chunk.rows)); + } + } _opts.meta->set_num_rows(_num_rows); - none_null_size = cast_set(non_null_value_size); - + none_null_size = non_null_value_size; + _chunks.clear(); _is_finalized = true; return Status::OK(); } From 5314b8588470a41db46f04a1b776507d7c99729a Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sun, 18 Oct 2026 14:05:49 +0800 Subject: [PATCH 4/7] [test](variant) Cover chunked VariantSubcolumnWriter edge cases (CIR-22037) --- be/src/storage/segment/column_writer.h | 8 + .../variant/variant_column_writer_impl.cpp | 41 ++- .../variant_column_writer_reader_test.cpp | 287 +++++++++++++++++- 3 files changed, 331 insertions(+), 5 deletions(-) diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 6e5dd9afad5c64..c1f5922c2d5759 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -643,6 +643,14 @@ class VariantSubcolumnWriter : public ColumnWriter { size_t get_non_null_size() const { return none_null_size; } +#ifdef BE_TEST + // Force small chunk budgets so tests can exercise multi-chunk flushes cheaply. + static void set_chunk_bytes_for_test(size_t bytes); + static void set_chunk_values_for_test(size_t values); + size_t num_chunks_for_test() const; + size_t max_chunk_bytes_for_test() const; +#endif + Status finalize(); private: diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 33201a6a4f9ae0..a47ef90a55a2e8 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -548,6 +548,11 @@ uint64_t VariantColumnWriterImpl::get_total_data_pages_bytes( return _v2_writer->get_total_data_pages_bytes(getter); } +#ifdef BE_TEST +static size_t g_chunk_bytes_for_test = 0; +static size_t g_chunk_values_for_test = 0; +#endif + VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column) : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { @@ -628,7 +633,19 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ RETURN_IF_ERROR(append_encoded(encoded_batch->read_view(), 0)); } _chunk_rows = _num_rows + num_rows - _chunk_row_begin; - if (_v2_builder->byte_size() >= kChunkBytes || _v2_builder->non_null_rows() >= kChunkValues) { + const size_t chunk_bytes = +#ifdef BE_TEST + g_chunk_bytes_for_test != 0 ? g_chunk_bytes_for_test : kChunkBytes; +#else + kChunkBytes; +#endif + const size_t chunk_values = +#ifdef BE_TEST + g_chunk_values_for_test != 0 ? g_chunk_values_for_test : kChunkValues; +#else + kChunkValues; +#endif + if (_v2_builder->byte_size() >= chunk_bytes || _v2_builder->non_null_rows() >= chunk_values) { RETURN_IF_ERROR(_seal_chunk()); } return Status::OK(); @@ -669,6 +686,28 @@ Status VariantSubcolumnWriter::append_data(const uint8_t** ptr, size_t num_rows) return _append(nullptr, ptr, num_rows); } +#ifdef BE_TEST +void VariantSubcolumnWriter::set_chunk_bytes_for_test(size_t bytes) { + g_chunk_bytes_for_test = bytes; +} + +void VariantSubcolumnWriter::set_chunk_values_for_test(size_t values) { + g_chunk_values_for_test = values; +} + +size_t VariantSubcolumnWriter::num_chunks_for_test() const { + return _chunks.size() + (_v2_builder != nullptr ? 1 : 0); +} + +size_t VariantSubcolumnWriter::max_chunk_bytes_for_test() const { + size_t max = _v2_builder != nullptr ? _v2_builder->byte_size() : 0; + for (const auto& chunk : _chunks) { + max = std::max(max, chunk.builder->byte_size()); + } + return max; +} +#endif + uint64_t VariantSubcolumnWriter::estimate_buffer_size() { if (is_finalized()) { return _writer ? _writer->estimate_buffer_size() : 0; diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index bedb522ec76d29..bed4a232194aa4 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -1327,6 +1327,25 @@ static Status create_variant_root_reader(const SegmentFooterPB& footer, class VariantColumnWriterReaderTest : public testing::Test { public: + // Parent VARIANT column with an extracted VARIANT subcolumn `v.payload`, i.e. the layout + // VariantSubcolumnWriter is responsible for. + void init_extracted_variant_schema(int64_t tablet_id) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb.add_column(), 1, "VARIANT", "v", 1); + _tablet_schema = std::make_shared(); + _tablet_schema->init_from_pb(schema_pb); + const TabletColumn& parent_column = _tablet_schema->column_by_uid(1); + TabletColumn extracted_column; + extracted_column.set_name(parent_column.name_lower_case() + ".payload"); + extracted_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + extracted_column.set_parent_unique_id(parent_column.unique_id()); + extracted_column.set_path_info(PathInData(parent_column.name_lower_case() + ".payload")); + extracted_column.set_is_nullable(true); + _tablet_schema->append_column(extracted_column); + init_tablet_from_current_schema(tablet_id); + } + void SetUp() override { // absolute dir char buffer[MAX_PATH_LEN]; @@ -1721,9 +1740,10 @@ class VariantColumnWriterReaderTest : public testing::Test { return Status::OK(); } - Status write_extracted_variant_segment(const ColumnPtr& source, const DataTypePtr& source_type, - std::string_view rowset_id, SegmentFooterPB* footer, - std::string* file_path, size_t first_batch_rows = 0) { + Status write_extracted_variant_segment( + const ColumnPtr& source, const DataTypePtr& source_type, std::string_view rowset_id, + SegmentFooterPB* footer, std::string* file_path, size_t first_batch_rows = 0, + size_t batch_rows = 0, const std::function& after_batch = {}) { DORIS_CHECK(source.get() != nullptr); DORIS_CHECK(source_type != nullptr); DORIS_CHECK(footer != nullptr); @@ -1768,11 +1788,28 @@ class VariantColumnWriterReaderTest : public testing::Test { converter.clear_source_content(0); return Status::OK(); }; - if (first_batch_rows > 0 && first_batch_rows < source->size()) { + if (batch_rows > 0) { + for (size_t row_pos = 0; row_pos < source->size(); row_pos += batch_rows) { + const size_t rows = std::min(batch_rows, source->size() - row_pos); + RETURN_IF_ERROR(append_batch(row_pos, rows)); + if (after_batch) { + after_batch(writer.get()); + } + } + } else if (first_batch_rows > 0 && first_batch_rows < source->size()) { RETURN_IF_ERROR(append_batch(0, first_batch_rows)); + if (after_batch) { + after_batch(writer.get()); + } RETURN_IF_ERROR(append_batch(first_batch_rows, source->size() - first_batch_rows)); + if (after_batch) { + after_batch(writer.get()); + } } else { RETURN_IF_ERROR(append_batch(0, source->size())); + if (after_batch) { + after_batch(writer.get()); + } } RETURN_IF_ERROR(writer->finish()); @@ -8973,4 +9010,246 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful EXPECT_EQ(actual_rows, expected_rows); } +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_large_values) { + // A large VARIANT subcolumn must be flushed in bounded chunks: a single segment-sized + // ColumnString is what overflows ColumnStr at 4 GiB. Compare the chunked output with + // the single-chunk reference and check the observed chunk budget. + init_extracted_variant_schema(11401); + constexpr size_t kRows = 3000; + constexpr size_t kBatchRows = 300; + const std::string value(1024, 'x'); + const std::string json = "\"" + value + "\""; + std::vector jsons(kRows, json); + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, + &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_chunk_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + ASSERT_EQ(ref_rows.size(), kRows); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(250); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + size_t max_chunks = 0; + size_t max_chunk_bytes = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_chunk_values", &chunked_footer, + &chunked_path, 0, kBatchRows, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + max_chunk_bytes = std::max(max_chunk_bytes, + subcolumn->max_chunk_bytes_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + // A chunk holds at most the configured budget plus the batch that crossed it. + EXPECT_LE(max_chunk_bytes, 250 * 1024 + kBatchRows * 1024); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + EXPECT_EQ(chunked_rows.front(), ref_rows.front()); + EXPECT_EQ(chunked_rows.back(), ref_rows.back()); +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_type_promotion) { + // Values that widen the inferred type across a chunk boundary (int -> string -> JSONB) must + // still produce the same column as the single-chunk path. + init_extracted_variant_schema(11402); + constexpr size_t kHalf = 2000; + std::vector jsons; + jsons.reserve(2 * kHalf); + for (size_t i = 0; i < kHalf; ++i) { + jsons.push_back(std::to_string(i)); + } + for (size_t i = 0; i < kHalf; ++i) { + jsons.push_back("\"s" + std::to_string(i) + "\""); + } + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, + &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_promote_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(500); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + size_t max_chunks = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_promote_chunked", &chunked_footer, + &chunked_path, 0, 300, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_sparse_rows) { + // Rows without the subcolumn become nulls. When chunks split a gap the dense materialization + // must still place every value on its logical row. + init_extracted_variant_schema(11403); + constexpr size_t kRows = 3000; + std::vector jsons(kRows, "\"v\""); + std::vector outer_nulls(kRows, 0); + for (size_t row = 0; row < kRows; row += 3) { + outer_nulls[row] = 1; + } + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, outer_nulls, + &source, &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_sparse_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(200); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + size_t max_chunks = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_sparse_chunked", &chunked_footer, + &chunked_path, 0, 300, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + ASSERT_FALSE(chunked_rows.empty()); + EXPECT_EQ(chunked_rows[0], "NULL"); + EXPECT_EQ(chunked_rows[1], ref_rows[1]); +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_all_null) { + // A subcolumn whose every row is null must still append the right number of nulls per chunk. + init_extracted_variant_schema(11404); + constexpr size_t kRows = 2000; + std::vector jsons(kRows, "\"v\""); + std::vector outer_nulls(kRows, 1); + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, outer_nulls, + &source, &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_null_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(100); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_null_chunked", + &chunked_footer, &chunked_path, 0, 300) + .ok()); + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + for (const auto& row : chunked_rows) { + EXPECT_EQ(row, "NULL"); + } +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_bytes) { + // The byte budget (kChunkBytes) seals a chunk even when the value count is small, e.g. few very + // wide strings. The chunk must still be flushed independently and read back unchanged. + init_extracted_variant_schema(11405); + constexpr size_t kRows = 3000; + constexpr size_t kBatchRows = 200; + const std::string value(1024, 'y'); + const std::string json = "\"" + value + "\""; + std::vector jsons(kRows, json); + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, + &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_bytes_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(256 * 1024); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(0); } + } guard; + size_t max_chunks = 0; + size_t max_chunk_bytes = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_bytes_chunked", &chunked_footer, + &chunked_path, 0, kBatchRows, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + max_chunk_bytes = std::max(max_chunk_bytes, + subcolumn->max_chunk_bytes_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + EXPECT_LE(max_chunk_bytes, 256 * 1024 + kBatchRows * 1024); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); +} + } // namespace doris From 720f7f7bc8979c92aae6cb7e6f43f72f06d5722b Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 20 Oct 2026 12:32:32 +0800 Subject: [PATCH 5/7] [fix](variant) Keep chunk type inference and ARRAY chunk dispatch consistent (CIR-22037) - A new chunk starts from the type inferred by the sealed chunks, so the single builder's JSONB fallback still applies across chunk boundaries (a bool after integers must not be coerced to 1/0 by the least-common-type merge). - All-NULL ARRAY chunks are flushed through the array writer, which updates the offsets, nulls and inverted index together; append_nulls only wrote the data column and left the index inconsistent. - One convertor per column in finalize: the ARRAY convertor carries the accumulated item offset, so chunked flushes keep absolute offsets. - Tests for int-then-bool across a boundary and an ARRAY all-NULL tail chunk. --- be/src/storage/segment/column_writer.h | 3 + .../variant/variant_column_writer_impl.cpp | 16 ++- .../variant/variant_writer_helpers.cpp | 32 ++++- .../segment/variant/variant_writer_helpers.h | 7 ++ .../variant_column_writer_reader_test.cpp | 113 +++++++++++++++++- 5 files changed, 156 insertions(+), 15 deletions(-) diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index c1f5922c2d5759..d6d0c939cf623f 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -678,6 +678,9 @@ class VariantSubcolumnWriter : public ColumnWriter { std::vector _chunks; size_t _chunk_row_begin = 0; size_t _chunk_rows = 0; + // Type accumulated by the sealed chunks. A new chunk starts from it so appends follow the + // exact same inference and JSONB-fallback sequence as one uninterrupted builder. + DataTypePtr _running_type; size_t _num_rows = 0; ColumnWriterOptions _opts; std::unique_ptr _writer; diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index a47ef90a55a2e8..5bbf977108faa3 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -602,6 +602,12 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ DORIS_CHECK(outer_nulls.empty() || outer_nulls.size() == num_rows); if (_v2_builder == nullptr) { RETURN_IF_ERROR(_initialize_v2_builder()); + if (_running_type != nullptr) { + // Continue from the type inferred by the sealed chunks. Appending a value that the + // previous chunks' type cannot hold (for example a bool after integers) then takes + // the builder's own JSONB fallback inside this chunk instead of being merged away. + RETURN_IF_ERROR(_v2_builder->convert_to(_running_type)); + } _chunk_row_begin = _num_rows; _chunk_rows = 0; } @@ -656,6 +662,7 @@ Status VariantSubcolumnWriter::_seal_chunk() { return Status::OK(); } RETURN_IF_ERROR(_v2_builder->complete_rows(_chunk_rows)); + _running_type = _v2_builder->type(); _chunks.push_back(ValueChunk { .builder = std::move(_v2_builder), .row_begin = _chunk_row_begin, .rows = _chunk_rows}); _v2_builder.reset(); @@ -803,18 +810,17 @@ Status VariantSubcolumnWriter::finalize() { _indexes, &opts, non_null_value_size, need_record_none_null_value_size)); _opts = opts; + // One convertor for the whole column. The ARRAY convertor carries its accumulated item offset + // across calls, so flushing chunk by chunk keeps the absolute array offsets correct; a fresh + // convertor per chunk would restart them at zero and corrupt the column. + OlapBlockDataConvertor converter; if (!publish_builder) { - OlapBlockDataConvertor converter; MutableColumnPtr defaults = flush_type->create_column(); RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( flush_column, _writer.get(), &converter, 0, flush_type, std::move(defaults), {}, _num_rows)); } else { for (auto& chunk : _chunks) { - // Each chunk owns one contiguous row range, so flushing it needs its own converter - // slot. Reuse is not possible: append_sparse_converted_column registers a convertor - // per call. - OlapBlockDataConvertor converter; if (chunk.builder->non_null_rows() == 0) { MutableColumnPtr empty = flush_type->create_column(); RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column( diff --git a/be/src/storage/segment/variant/variant_writer_helpers.cpp b/be/src/storage/segment/variant/variant_writer_helpers.cpp index f5cf7d9ab3b35b..a414e772e2faba 100644 --- a/be/src/storage/segment/variant/variant_writer_helpers.cpp +++ b/be/src/storage/segment/variant/variant_writer_helpers.cpp @@ -18,6 +18,7 @@ #include "storage/segment/variant/variant_writer_helpers.h" #include +#include #include #include #include @@ -322,6 +323,20 @@ Status append_sparse_scalar_column(const TabletColumn& tablet_column, ColumnWrit } // namespace +#ifdef BE_TEST +namespace { +std::atomic g_array_all_null_chunks_for_test = 0; +} // namespace + +uint64_t array_all_null_chunks_for_test() { + return g_array_all_null_chunks_for_test.load(std::memory_order_relaxed); +} + +void reset_array_all_null_chunks_for_test() { + g_array_all_null_chunks_for_test.store(0, std::memory_order_relaxed); +} +#endif + Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnWriter* writer, OlapBlockDataConvertor* converter, int column_id, const DataTypePtr& type, const ColumnPtr& values_column, @@ -338,16 +353,23 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW if (total_rows == 0) { return Status::OK(); } - if (rowids.empty()) { - DORIS_CHECK(tablet_column.is_nullable()); - return writer->append_nulls(total_rows); - } - const DataTypePtr base_type = remove_nullable(type); if (base_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY) { +#ifdef BE_TEST + if (rowids.empty()) { + g_array_all_null_chunks_for_test.fetch_add(1, std::memory_order_relaxed); + } +#endif + // ARRAY writers materialize a dense N-row column and update offsets, nulls and the + // inverted index together. An all-NULL chunk (empty rowids) must take this path too: + // append_nulls only writes column data and leaves the inverted index inconsistent. return append_sparse_array_column(tablet_column, writer, converter, column_id, type, values_column, rowids, total_rows); } + if (rowids.empty()) { + DORIS_CHECK(tablet_column.is_nullable()); + return writer->append_nulls(total_rows); + } return append_sparse_scalar_column(tablet_column, writer, converter, column_id, type, values_column, rowids, total_rows); } diff --git a/be/src/storage/segment/variant/variant_writer_helpers.h b/be/src/storage/segment/variant/variant_writer_helpers.h index 5caea664990434..98722b9f808e84 100644 --- a/be/src/storage/segment/variant/variant_writer_helpers.h +++ b/be/src/storage/segment/variant/variant_writer_helpers.h @@ -76,6 +76,13 @@ Status convert_and_write_column(OlapBlockDataConvertor* converter, const TabletC // Converts only present values. Missing logical rows are appended as NULL; ARRAY uses a one-path // full-column fallback because its converted representation is not fixed-width/strided. +#ifdef BE_TEST +// Number of all-NULL ARRAY chunks written through the array path. The dispatch must never send +// them to the scalar append_nulls fast path, which does not maintain the inverted index. +uint64_t array_all_null_chunks_for_test(); +void reset_array_all_null_chunks_for_test(); +#endif + Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnWriter* writer, OlapBlockDataConvertor* converter, int column_id, const DataTypePtr& type, const ColumnPtr& values_column, diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index bed4a232194aa4..97bcddb1e00fe4 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -47,6 +47,7 @@ #include "core/value/uuid_value.h" #include "core/value/variant/variant_batch_builder.h" #include "core/value/variant/variant_parquet_encoding.h" +#include "exec/common/variant_util.h" #include "exprs/function/search/function_search.h" #include "exprs/function/search/variant_inverted_index_search.h" #include "exprs/vectorized_fn_call.h" @@ -79,6 +80,7 @@ #include "storage/segment/variant/variant_column_reader.h" #include "storage/segment/variant/variant_column_writer_impl.h" #include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h" +#include "storage/segment/variant/variant_writer_helpers.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_manager.h" #include "testutil/variant_util.h" @@ -1872,9 +1874,13 @@ class VariantColumnWriterReaderTest : public testing::Test { RETURN_IF_ERROR(ColumnReader::create(reader_options, footer.columns(0), footer.num_rows(), file_reader, &reader)); - TabletColumn read_column = _tablet_schema->column(1); - read_column.set_type(static_cast(footer.columns(0).type())); - auto read_type = DataTypeFactory::instance().create_data_type(read_column, false); + // Build the read type and the matching tablet column from the footer meta so nested types + // (ARRAY/MAP/STRUCT) keep their child type information; the schema column only carries the + // top-level type and the array iterator needs the element sub-column. + auto read_type = DataTypeFactory::instance().create_data_type(footer.columns(0)); + TabletColumn read_column = variant_util::get_column_by_type( + read_type, _tablet_schema->column(1).name(), + variant_util::ExtraInfo {.unique_id = -1, .parent_unique_id = 1}); ColumnIteratorUPtr iterator; RETURN_IF_ERROR(reader->new_iterator(&iterator, &read_column)); OlapReaderStatistics stats; @@ -2489,8 +2495,9 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_root_writer_streams_per_batch converter.clear_source_content(0); max_buffered_bytes = std::max(max_buffered_bytes, writer->estimate_buffer_size()); } - // The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB); the - // streaming writer holds at most the current page plus the outer-null map. + // The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB). The + // streaming writer flushes the root column batch by batch, so its own retained raw buffer is + // about one batch; encoded pages and the other variant column writers are separate. EXPECT_LT(max_buffered_bytes, 2 * kBatchRows * kValueBytes); ASSERT_TRUE(writer->finish().ok()); @@ -9252,4 +9259,100 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_by EXPECT_EQ(chunked_rows, ref_rows); } +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_int_then_bool) { + // The single builder falls back to JSONB when a bool follows integers: appending `true` into + // an Int64 column is not a representable promotion. Chunks must carry the inferred type across + // their boundary so the same value sequence keeps the JSONB fallback instead of coercing + // true/false to 1/0 through the least-common-type merge. + init_extracted_variant_schema(11406); + const std::vector jsons {"7", "true", "false", "8"}; + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, + &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_bool_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + size_t max_chunks = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_bool_chunked", &chunked_footer, + &chunked_path, 0, 1, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + EXPECT_EQ(chunked_rows, (std::vector {"7", "true", "false", "8"})); + EXPECT_EQ(static_cast(ref_footer.columns(0).type()), + FieldType::OLAP_FIELD_TYPE_JSONB); + EXPECT_EQ(static_cast(chunked_footer.columns(0).type()), + FieldType::OLAP_FIELD_TYPE_JSONB); +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_array_all_null_tail) { + // An all-NULL tail chunk of an ARRAY subcolumn must be flushed through the array writer (which + // updates offsets, nulls and the inverted index together), not through append_nulls. + init_extracted_variant_schema(11407); + const std::vector jsons {"[1,2]", "[\"a\",\"b\"]", "null", "null"}; + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source, + &source_type) + .ok()); + + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_array_ref", + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(2); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + segment_v2::variant_writer_helpers::reset_array_all_null_chunks_for_test(); + size_t max_chunks = 0; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment( + source, source_type, "subcolumn_array_chunked", &chunked_footer, + &chunked_path, 0, 1, + [&](ColumnWriter* writer) { + auto* subcolumn = dynamic_cast(writer); + ASSERT_NE(subcolumn, nullptr); + max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test()); + }) + .ok()); + EXPECT_GE(max_chunks, 2); + EXPECT_EQ(segment_v2::variant_writer_helpers::array_all_null_chunks_for_test(), 1u); + + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + ASSERT_EQ(chunked_rows.size(), 4); + EXPECT_EQ(chunked_rows[2], "NULL"); + EXPECT_EQ(chunked_rows[3], "NULL"); +} + } // namespace doris From cf1db49b0d97aadc1f0110fae66c828e2e059bd7 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 20 Oct 2026 15:25:04 +0800 Subject: [PATCH 6/7] [test](variant) Reproduce decimal inference across writer chunks ### What problem does this PR solve? Related PR: #69221 Problem Summary: Chunked extracted VARIANT writes can discard valid Decimal values or reduce decimal-array scale. Add native Decimal writer/readback cases comparing chunked and uninterrupted inference, including a later Double and values retained in the active chunk, plus a direct nullable-array type-merge check. ### Release note None ### Check List (For Author) - Test: ASAN BE unit tests: all three added tests fail on the unmodified implementation (red); clang-format 16 and build hygiene pass - Behavior changed: No (tests only) - Does this need documentation: No --- .../variant_column_writer_reader_test.cpp | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 97bcddb1e00fe4..5f0823be094ad7 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -9355,4 +9355,137 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_array EXPECT_EQ(chunked_rows[3], "NULL"); } +TEST(VariantPathBuilderTest, MergeNullableDecimalArraysPreservesScale) { + VariantBatchBuilder value_builder; + for (size_t index = 0; index < 2; ++index) { + auto row = value_builder.begin_row(); + auto array = row.start_array(); + row.add_decimal(index == 0 ? 123 : 12345, index == 0 ? 2 : 4, 16); + array.finish(); + row.finish(); + } + const auto values = value_builder.finish_batch(); + segment_v2::VariantPathBuilder first(PathInData("metric")); + segment_v2::VariantPathBuilder second(PathInData("metric")); + ASSERT_TRUE(first.append(values.value_at(0), 0).ok()); + ASSERT_TRUE(second.append(values.value_at(1), 0).ok()); + for (bool reverse : {false, true}) { + const DataTypePtr common = segment_v2::variant_path_least_common_type( + reverse ? second.type() : first.type(), reverse ? first.type() : second.type()); + EXPECT_TRUE(common->is_nullable()); + const auto& array = assert_cast(*remove_nullable(common)); + EXPECT_EQ(array.get_nested_type()->get_scale(), 4); + } +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_decimal_overflow) { + init_extracted_variant_schema(11408); + for (bool array_values : {false, true}) { + for (bool double_tail : {false, true}) { + SCOPED_TRACE(testing::Message() + << "array=" << array_values << ", double_tail=" << double_tail); + VariantBatchBuilder value_builder; + for (size_t index = 0; index < (double_tail ? 5 : 4); ++index) { + auto row = value_builder.begin_row(); + std::optional array; + if (array_values) { + array.emplace(row.start_array()); + } + if (index == 1) { + row.add_decimal(static_cast<__int128>(VARIANT_DECIMAL16_MAX), 1, 16); + } else if (index == 3) { + row.add_decimal(1, 38, 16); + } else if (index == 4) { + row.add_double(2.5); + } else { + row.add_decimal(1, 1, 16); + } + if (array_values) { + array->finish(); + } + row.finish(); + } + auto source = ColumnVariantV2::create(); + source->insert_encoded_batch(value_builder.finish_batch()); + const auto source_type = std::make_shared(0, false); + const auto suffix = fmt::format("{}_{}", array_values, double_tail); + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type, + "decimal_overflow_ref_" + suffix, + &ref_footer, &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + ASSERT_EQ(static_cast(ref_footer.columns(0).type()), + FieldType::OLAP_FIELD_TYPE_JSONB); + + struct ChunkGuard { + explicit ChunkGuard(size_t values) { + VariantSubcolumnWriter::set_chunk_values_for_test(values); + } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard(double_tail ? 2 : 1); + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type, + "decimal_overflow_chunks_" + suffix, + &chunked_footer, &chunked_path, 0, 1) + .ok()); + std::vector chunked_rows; + ASSERT_TRUE( + read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + // The scale-38 value cannot widen the earlier large value losslessly. With two rows + // per chunk, fallback must also preserve the preceding 0.1 in the active chunk. + EXPECT_EQ(chunked_rows, ref_rows); + EXPECT_EQ(static_cast(chunked_footer.columns(0).type()), + FieldType::OLAP_FIELD_TYPE_JSONB); + } + } +} + +TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_decimal_array_scales) { + init_extracted_variant_schema(11409); + for (bool reverse : {false, true}) { + SCOPED_TRACE(testing::Message() << "reverse=" << reverse); + VariantBatchBuilder value_builder; + for (size_t index = 0; index < 2; ++index) { + auto row = value_builder.begin_row(); + auto array = row.start_array(); + const bool high_scale = (index == 1) != reverse; + row.add_decimal(high_scale ? 12345 : 123, high_scale ? 4 : 2, 16); + array.finish(); + row.finish(); + } + auto source = ColumnVariantV2::create(); + source->insert_encoded_batch(value_builder.finish_batch()); + const auto source_type = std::make_shared(0, false); + const auto suffix = std::to_string(reverse); + SegmentFooterPB ref_footer; + std::string ref_path; + ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type, + "decimal_array_ref_" + suffix, &ref_footer, + &ref_path) + .ok()); + std::vector ref_rows; + ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok()); + ASSERT_EQ(ref_footer.columns(0).children_columns(0).frac(), 4); + + struct ChunkGuard { + ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); } + ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); } + } guard; + SegmentFooterPB chunked_footer; + std::string chunked_path; + ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type, + "decimal_array_chunks_" + suffix, + &chunked_footer, &chunked_path, 0, 1) + .ok()); + std::vector chunked_rows; + ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok()); + EXPECT_EQ(chunked_rows, ref_rows); + EXPECT_EQ(chunked_footer.columns(0).children_columns(0).frac(), 4); + } +} + } // namespace doris From fab1e0c9db61ab0096045b63fa3a570d5562db60 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 20 Oct 2026 15:44:50 +0800 Subject: [PATCH 7/7] [fix](variant) Preserve inferred values across writer chunks ### What problem does this PR solve? Related PR: #69221 Problem Summary: Extracted VARIANT chunks inherit a type without validating it against earlier values. Decimal widening can therefore filter valid rows as cast NULLs, and nullable-array type merging can reduce element scale. Resolve inferred promotions against retained chunks before committing any cast, preserving whole-path JSONB fallback and the active chunk's original Decimal representation. Unwrap nullable types before array/Decimal dispatch. Keep explicit typed-storage cast-null filtering unchanged. ### Release note Preserve VARIANT Decimal values and decimal-array precision during chunked writes. ### Check List (For Author) - Test: ASAN BE unit tests: three new tests fail before the fix and pass afterward; related writer, path-builder, shredder and compatibility tests: 103 passed, 2 skipped because NestedGroup write support is unavailable. Local UT uses GLIBC_COMPATIBILITY=OFF to avoid duplicate toolchain compatibility symbols. clang-format 16, build hygiene and clang-tidy on changed lines passed. - Behavior changed: Yes, inferred cross-chunk conversions preserve valid values and Decimal array scale - Does this need documentation: No --- be/src/storage/segment/column_writer.h | 5 +- .../variant/v2/variant_path_builder.cpp | 71 ++++++++++++++----- .../segment/variant/v2/variant_path_builder.h | 13 +++- .../variant/variant_column_writer_impl.cpp | 32 ++++++++- 4 files changed, 98 insertions(+), 23 deletions(-) diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index d6d0c939cf623f..0a2ef6dcbe50d0 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -659,6 +659,7 @@ class VariantSubcolumnWriter : public ColumnWriter { std::span outer_nulls); Status _ensure_input_format(const VariantColumnData& column); Status _initialize_v2_builder(); + Status _promote_chunks(DataTypePtr* inferred_type); Status _seal_chunk(); // Bounded value chunks; each chunk is finalized and flushed independently so the writer never // holds a segment-sized ColumnString. @@ -678,8 +679,8 @@ class VariantSubcolumnWriter : public ColumnWriter { std::vector _chunks; size_t _chunk_row_begin = 0; size_t _chunk_rows = 0; - // Type accumulated by the sealed chunks. A new chunk starts from it so appends follow the - // exact same inference and JSONB-fallback sequence as one uninterrupted builder. + // Type carried into the next chunk. Inferred promotions also check and update sealed chunks + // so fallback follows the same sequence as one uninterrupted builder. DataTypePtr _running_type; size_t _num_rows = 0; ColumnWriterOptions _opts; diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp index 42083f4ead4ba8..77757244d6c85d 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp +++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp @@ -371,6 +371,11 @@ DataTypePtr path_least_common_type(const DataTypePtr& left, const DataTypePtr& r if (left.get() == right.get() || left->equals(*right)) { return left; } + // Chunk builders expose Nullable(Array(...)). Unwrap before dispatching so decimal array + // elements use the scale-aware rules below instead of the generic first-primitive match. + if (left->is_nullable() || right->is_nullable()) { + return make_nullable(path_least_common_type(remove_nullable(left), remove_nullable(right))); + } const auto* left_array = typeid_cast(left.get()); const auto* right_array = typeid_cast(right.get()); if (left_array != nullptr || right_array != nullptr) { @@ -1082,39 +1087,54 @@ struct VariantPathBuilder::Impl { return Status::OK(); } - Status promote(const DataTypePtr& target_type, bool filter_cast_nulls) { - DataTypePtr target = remove_nullable(target_type); - if (target->equals(*type)) { - return Status::OK(); - } - ColumnPtr promoted; + Status cast_to_type(const DataTypePtr& target, ColumnPtr* promoted) const { if (target->get_primitive_type() == TYPE_STRING && type->get_primitive_type() == TYPE_ARRAY) { RETURN_IF_ERROR(stringify_complex_column( - type, assert_cast(*column), &promoted)); + type, assert_cast(*column), promoted)); } else if (type->get_primitive_type() == TYPE_ARRAY && target->get_primitive_type() == TYPE_ARRAY && variant_path_type_contains_nothing(type)) { MutableColumnPtr materialized; RETURN_IF_ERROR(replace_array_nothing(nullable_type, *column, make_nullable(target), &materialized)); - promoted = std::move(materialized); + *promoted = std::move(materialized); } else { RETURN_IF_ERROR( variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()}, - make_nullable(target), &promoted)); + make_nullable(target), promoted)); } + return Status::OK(); + } + + Status promote(const DataTypePtr& target_type, bool filter_cast_nulls, + const InferredPromotion& on_promotion = {}) { + DataTypePtr target = remove_nullable(target_type); + if (target->equals(*type)) { + return Status::OK(); + } + ColumnPtr promoted; + RETURN_IF_ERROR(cast_to_type(target, &promoted)); // Inferred widening is only a storage representation choice. If CAST loses a valid value // at any array depth, preserve the whole path as JSONB instead. Forced typed-path // conversion intentionally retains its existing cast-null filtering below. if (!filter_cast_nulls && target->get_primitive_type() != TYPE_JSONB && cast_introduced_null(*column, *promoted)) { target = jsonb_type(); - RETURN_IF_ERROR( - variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()}, - make_nullable(target), &promoted)); + RETURN_IF_ERROR(cast_to_type(target, &promoted)); DORIS_CHECK(!assert_cast(*promoted).has_null()); } + if (on_promotion) { + const DataTypePtr proposed = target; + RETURN_IF_ERROR(on_promotion(&target)); + target = remove_nullable(target); + if (!target->equals(*proposed)) { + // For example, the active chunk's 0.1 fits scale 38, but a sealed large Decimal + // does not. Recast the original active values to JSONB, preserving 0.1's scale. + DORIS_CHECK_EQ(target->get_primitive_type(), TYPE_JSONB); + RETURN_IF_ERROR(cast_to_type(target, &promoted)); + } + } const auto& nullable = assert_cast(*promoted); if (nullable.has_null()) { DORIS_CHECK(filter_cast_nulls); @@ -1164,7 +1184,8 @@ VariantPathBuilder::~VariantPathBuilder() = default; VariantPathBuilder::VariantPathBuilder(VariantPathBuilder&&) noexcept = default; VariantPathBuilder& VariantPathBuilder::operator=(VariantPathBuilder&&) noexcept = default; -Status VariantPathBuilder::append(VariantRef value, size_t row) { +Status VariantPathBuilder::append(VariantRef value, size_t row, + const InferredPromotion& on_promotion) { try { const ScalarPhysical physical = scalar_physical(value); if (physical.kind == ScalarPhysicalKind::NULL_VALUE) { @@ -1193,14 +1214,14 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) { DataTypePtr common_type = path_least_common_type(_impl->type, inferred_type); if (_impl->type.get() != common_type.get() && !_impl->type->equals(*common_type)) { - RETURN_IF_ERROR(_impl->promote(common_type, false)); + RETURN_IF_ERROR(_impl->promote(common_type, false, on_promotion)); } } } } const bool is_array = !stable_scalar && value_kind(value) == ValueKind::ARRAY; if (is_array && !value_is_representable(value, _impl->type)) { - RETURN_IF_ERROR(_impl->promote(jsonb_type(), false)); + RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion)); } try { @@ -1211,7 +1232,7 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) { if (is_array || _impl->type->get_primitive_type() == TYPE_JSONB) { throw; } - RETURN_IF_ERROR(_impl->promote(jsonb_type(), false)); + RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion)); auto& nullable = assert_cast(*_impl->column); append_value(value, _impl->type, &nullable.get_nested_column()); nullable.get_null_map_data().push_back(0); @@ -1236,11 +1257,25 @@ Status VariantPathBuilder::complete_rows(size_t rows) { return Status::OK(); } -Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type) { +Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls) { if (!_impl->column) { return _impl->initialize(storage_type); } - return _impl->promote(storage_type, true); + return _impl->promote(storage_type, filter_cast_nulls); +} + +Status VariantPathBuilder::check_inferred_promotion(const DataTypePtr& target_type, + bool* needs_jsonb) const { + const DataTypePtr target = remove_nullable(target_type); + *needs_jsonb = false; + if (!_impl->column || target->equals(*_impl->type) || + target->get_primitive_type() == TYPE_JSONB) { + return Status::OK(); + } + ColumnPtr promoted; + RETURN_IF_ERROR(_impl->cast_to_type(target, &promoted)); + *needs_jsonb = cast_introduced_null(*_impl->column, *promoted); + return Status::OK(); } const PathInData& VariantPathBuilder::path() const { diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.h b/be/src/storage/segment/variant/v2/variant_path_builder.h index b48ab5f2ac55d4..cc21594840a06a 100644 --- a/be/src/storage/segment/variant/v2/variant_path_builder.h +++ b/be/src/storage/segment/variant/v2/variant_path_builder.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -54,6 +55,10 @@ bool variant_path_type_contains_nothing(const DataTypePtr& type); // returns. class VariantPathBuilder final { public: + // Resolve inferred widening against values held outside this builder, before either side + // commits its cast. The resolver may replace the proposed type with JSONB. + using InferredPromotion = std::function; + explicit VariantPathBuilder(PathInData path, size_t prefix_rows = 0); ~VariantPathBuilder(); @@ -62,9 +67,13 @@ class VariantPathBuilder final { VariantPathBuilder(const VariantPathBuilder&) = delete; VariantPathBuilder& operator=(const VariantPathBuilder&) = delete; - Status append(VariantRef value, size_t row); + Status append(VariantRef value, size_t row, const InferredPromotion& on_promotion = {}); Status complete_rows(size_t rows); - Status convert_to(const DataTypePtr& storage_type); + // Inferred conversions preserve values through JSONB fallback; explicit storage coercion + // retains the default cast-null filtering. + Status convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls = true); + // Probe inferred widening without changing values, types or rowids in retained chunks. + Status check_inferred_promotion(const DataTypePtr& target_type, bool* needs_jsonb) const; const PathInData& path() const; const DataTypePtr& type() const; diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 5bbf977108faa3..46796d27a85087 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -612,6 +612,11 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ _chunk_rows = 0; } + VariantPathBuilder::InferredPromotion on_promotion; + // Until a chunk is sealed, the active builder already holds every value to be checked. + if (!_chunks.empty()) { + on_promotion = [this](DataTypePtr* type) { return _promote_chunks(type); }; + } const auto append_encoded = [&](const ColumnVariantV2::ReadView& view, size_t begin) -> Status { DORIS_CHECK(!view.is_typed()); for (size_t offset = 0; offset < num_rows; ++offset) { @@ -621,7 +626,8 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ const size_t input_row = begin + offset; const VariantRef value = view.value_at(input_row); if (!value.is_null()) { - RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin)); + RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin, + on_promotion)); } } return Status::OK(); @@ -657,6 +663,30 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_ return Status::OK(); } +Status VariantSubcolumnWriter::_promote_chunks(DataTypePtr* inferred_type) { + // Decide against every retained value before changing any chunk. A small scale-1 Decimal may + // fit scale 38 while a later large Decimal does not; committing the first cast would change + // its JSONB representation before discovering that the whole path needs fallback. + for (const auto& chunk : _chunks) { + if (chunk.builder->non_null_rows() == 0) { + continue; + } + bool needs_jsonb = false; + RETURN_IF_ERROR(chunk.builder->check_inferred_promotion(*inferred_type, &needs_jsonb)); + if (needs_jsonb) { + *inferred_type = std::make_shared(); + break; + } + } + for (auto& chunk : _chunks) { + if (chunk.builder->non_null_rows() != 0) { + RETURN_IF_ERROR( + chunk.builder->convert_to(*inferred_type, false /* filter_cast_nulls */)); + } + } + return Status::OK(); +} + Status VariantSubcolumnWriter::_seal_chunk() { if (_v2_builder == nullptr) { return Status::OK();