Skip to content
Merged
25 changes: 25 additions & 0 deletions be/src/storage/segment/column_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,14 @@ class VariantSubcolumnWriter : public ColumnWriter {

size_t get_non_null_size() const { return none_null_size; }

#ifdef BE_TEST
// Force small chunk budgets so tests can exercise multi-chunk flushes cheaply.
static void set_chunk_bytes_for_test(size_t bytes);
static void set_chunk_values_for_test(size_t values);
size_t num_chunks_for_test() const;
size_t max_chunk_bytes_for_test() const;
#endif

Status finalize();

private:
Expand All @@ -651,12 +659,29 @@ class VariantSubcolumnWriter : public ColumnWriter {
std::span<const uint8_t> outer_nulls);
Status _ensure_input_format(const VariantColumnData& column);
Status _initialize_v2_builder();
Status _promote_chunks(DataTypePtr* inferred_type);
Status _seal_chunk();
// Bounded value chunks; each chunk is finalized and flushed independently so the writer never
// holds a segment-sized ColumnString.
struct ValueChunk {
std::unique_ptr<VariantPathBuilder> builder;
size_t row_begin = 0;
size_t rows = 0;
};
static constexpr size_t kChunkBytes = 256ULL << 20;
static constexpr size_t kChunkValues = 4ULL << 20;
bool is_finalized() const;
bool _is_finalized = false;
ordinal_t _next_rowid = 0;
size_t none_null_size = 0;
VariantWriterInputFormat _input_format = VariantWriterInputFormat::UNSET;
std::unique_ptr<VariantPathBuilder> _v2_builder;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

直接把这个 换成 std::vector<std::unique_ptr<VariantPathBuilder> _v2_builder> 是不是就简单了,compaction 是顺序写的 row_begin rows 这些 也不用记

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

顺序写,所以不用记录 rows 这个不完全成立:NULL 占据的逻辑行数仍要记录,只是可以由 builder 统一负责。跨 chunk 类型提升和 JSONB 回退也需要这个。

std::vector<ValueChunk> _chunks;
size_t _chunk_row_begin = 0;
size_t _chunk_rows = 0;
// Type carried into the next chunk. Inferred promotions also check and update sealed chunks
// so fallback follows the same sequence as one uninterrupted builder.
DataTypePtr _running_type;
size_t _num_rows = 0;
ColumnWriterOptions _opts;
std::unique_ptr<ColumnWriter> _writer;
Expand Down
41 changes: 31 additions & 10 deletions be/src/storage/segment/variant/v2/variant_column_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,12 @@ Status VariantV2ColumnWriter::init() {
_root_only = variant_writer_helpers::has_extracted_variant_columns(
*_opts.rowset_ctx->tablet_schema, _tablet_column->unique_id());
if (_root_only) {
_root_jsonb = ColumnString::create();
// Stream the root column instead of buffering a whole segment: the JSONB ColumnString
// would otherwise have to hold every row of the segment and could exceed the
// ColumnStr<uint32_t> 4 GiB limit before finalize.
_root_writer = std::make_unique<ScalarColumnWriter>(
_opts, std::make_shared<TabletColumn>(*_tablet_column), _opts.file_writer);
RETURN_IF_ERROR(_root_writer->init());
return Status::OK();
}

Expand All @@ -129,15 +134,14 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num
return Status::InternalError("Cannot append ColumnVariantV2 after writer finalization");
}
if (_root_only) {
DORIS_CHECK(_root_jsonb);
DORIS_CHECK(_root_writer);
if (num_rows > std::numeric_limits<size_t>::max() - _num_rows) {
return Status::InvalidArgument("Variant writer row count overflows size_t");
}
ColumnString::MutablePtr batch_root_jsonb;
RETURN_IF_ERROR(build_root_only_batch(column, num_rows, outer_nulls, &batch_root_jsonb));
DORIS_CHECK_EQ(batch_root_jsonb->size(), num_rows);
RETURN_IF_CATCH_EXCEPTION(
{ _root_jsonb->insert_range_from(*batch_root_jsonb, 0, num_rows); });
RETURN_IF_ERROR(_append_root_batch(*batch_root_jsonb, outer_nulls));
} else {
RETURN_IF_ERROR(variant_writer_helpers::append_variant_v2_to_shredder(
_shredder.get(), column, num_rows, outer_nulls));
Expand All @@ -152,6 +156,25 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num
return Status::OK();
}

Status VariantV2ColumnWriter::_append_root_batch(const ColumnString& batch,
std::span<const uint8_t> outer_nulls) {
DORIS_CHECK(_root_writer != nullptr);
if (batch.size() == 0) {
return Status::OK();
}
if (!outer_nulls.empty() && outer_nulls.size() != batch.size()) {
return Status::InvalidArgument("Variant root outer-null span has {} rows, expected {}",
outer_nulls.size(), batch.size());
}
DorisVector<Slice> slices(batch.size());
for (size_t row = 0; row < batch.size(); ++row) {
slices[row] = batch.get_data_at(row).to_slice();
}
const uint8_t* null_map =
_tablet_column->is_nullable() && !outer_nulls.empty() ? outer_nulls.data() : nullptr;
return _root_writer->append(null_map, slices.data(), batch.size());
}

Status VariantV2ColumnWriter::_write_root(const IColumn* root_jsonb, int& column_id) {
if (!root_jsonb || root_jsonb->size() != _num_rows) {
return Status::InvalidArgument("Variant root column has {} rows, expected {}",
Expand Down Expand Up @@ -231,10 +254,8 @@ Status VariantV2ColumnWriter::finalize() {
}
if (_root_only) {
DORIS_CHECK(_shredder == nullptr);
DORIS_CHECK(_root_jsonb);
int column_id = 0;
RETURN_IF_ERROR(_write_root(_root_jsonb.get(), column_id));
_root_jsonb.reset();
DORIS_CHECK(_root_writer != nullptr);
_opts.meta->set_num_rows(_num_rows);
_is_finalized = true;
return Status::OK();
}
Expand Down Expand Up @@ -343,8 +364,8 @@ Status VariantV2ColumnWriter::write_bloom_filter_index() {
uint64_t VariantV2ColumnWriter::estimate_buffer_size() {
if (!_is_finalized) {
if (_root_only) {
DORIS_CHECK(_root_jsonb);
return _outer_nulls->byte_size() + _root_jsonb->byte_size();
DORIS_CHECK(_root_writer != nullptr);
return _outer_nulls->byte_size() + _root_writer->estimate_buffer_size();
}
DORIS_CHECK(_shredder != nullptr);
return _outer_nulls->byte_size() + _shredder->byte_size();
Expand Down
9 changes: 5 additions & 4 deletions be/src/storage/segment/variant/v2/variant_column_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ class VariantBinaryWriter;
class VariantShredder;
struct VariantShreddedColumns;

// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append;
// a parent with extracted children retains only its root JSONB column. The writer never retains a
// segment-sized copy of the input execution column.
// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append.
// A parent with extracted children streams its root JSONB column into the root column writer batch
// by batch. The writer never retains a segment-sized copy of the input execution column.
class VariantV2ColumnWriter {
public:
VariantV2ColumnWriter(ColumnWriterOptions opts, const TabletColumn* column);
Expand All @@ -65,6 +65,8 @@ class VariantV2ColumnWriter {

private:
Status _write_root(const IColumn* root_jsonb, int& column_id);
// Streams one batch of the root-only JSONB column into the root column writer.
Status _append_root_batch(const ColumnString& batch, std::span<const uint8_t> outer_nulls);
Status _write_materialized(const VariantShreddedColumns& shredded,
OlapBlockDataConvertor* converter, int& column_id);
Status _write_binary(const VariantShreddedColumns& shredded, OlapBlockDataConvertor* converter,
Expand All @@ -74,7 +76,6 @@ class VariantV2ColumnWriter {
ColumnWriterOptions _opts;
const TabletColumn* _tablet_column = nullptr;
std::unique_ptr<VariantShredder> _shredder;
ColumnString::MutablePtr _root_jsonb;
ColumnUInt8::MutablePtr _outer_nulls;
size_t _num_rows = 0;
bool _root_only = false;
Expand Down
75 changes: 57 additions & 18 deletions be/src/storage/segment/variant/v2/variant_path_builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,11 @@ DataTypePtr path_least_common_type(const DataTypePtr& left, const DataTypePtr& r
if (left.get() == right.get() || left->equals(*right)) {
return left;
}
// Chunk builders expose Nullable(Array(...)). Unwrap before dispatching so decimal array
// elements use the scale-aware rules below instead of the generic first-primitive match.
if (left->is_nullable() || right->is_nullable()) {
return make_nullable(path_least_common_type(remove_nullable(left), remove_nullable(right)));
}
const auto* left_array = typeid_cast<const DataTypeArray*>(left.get());
const auto* right_array = typeid_cast<const DataTypeArray*>(right.get());
if (left_array != nullptr || right_array != nullptr) {
Expand Down Expand Up @@ -1038,6 +1043,10 @@ bool cast_introduced_null(const IColumn& source, const IColumn& target) {

} // namespace

DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right) {
return path_least_common_type(left, right);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Unwrap nullable arrays before merging chunk types

VariantPathBuilder::type() returns Nullable(...), so finalization passes nullable arrays through this wrapper. The internal path_least_common_type checks for DataTypeArray directly; those checks miss the nullable wrapper and delegate to get_least_supertype_jsonb, whose recursive decimal merge selects the first scale. If an untyped extracted path has [1.23] (scale 2) in an earlier chunk and [1.2345] (scale 4) in a later chunk, the second builder correctly widens to scale 4, but this final merge chooses scale 2 and the final cast changes [1.2345] to [1.23]. Single-builder inference retains scale 4. Unwrap the outer nullable types before dispatching so array elements use the metadata-aware common-type rules, and cover this with a multi-chunk decimal-array test.

}

DataTypePtr normalize_variant_path_integer_widths(const DataTypePtr& type) {
const DataTypePtr base = remove_nullable(type);
if (const auto* array = typeid_cast<const DataTypeArray*>(base.get())) {
Expand Down Expand Up @@ -1078,39 +1087,54 @@ struct VariantPathBuilder::Impl {
return Status::OK();
}

Status promote(const DataTypePtr& target_type, bool filter_cast_nulls) {
DataTypePtr target = remove_nullable(target_type);
if (target->equals(*type)) {
return Status::OK();
}
ColumnPtr promoted;
Status cast_to_type(const DataTypePtr& target, ColumnPtr* promoted) const {
if (target->get_primitive_type() == TYPE_STRING &&
type->get_primitive_type() == TYPE_ARRAY) {
RETURN_IF_ERROR(stringify_complex_column(
type, assert_cast<const ColumnNullable&>(*column), &promoted));
type, assert_cast<const ColumnNullable&>(*column), promoted));
} else if (type->get_primitive_type() == TYPE_ARRAY &&
target->get_primitive_type() == TYPE_ARRAY &&
variant_path_type_contains_nothing(type)) {
MutableColumnPtr materialized;
RETURN_IF_ERROR(replace_array_nothing(nullable_type, *column, make_nullable(target),
&materialized));
promoted = std::move(materialized);
*promoted = std::move(materialized);
} else {
RETURN_IF_ERROR(
variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()},
make_nullable(target), &promoted));
make_nullable(target), promoted));
}
return Status::OK();
}

Status promote(const DataTypePtr& target_type, bool filter_cast_nulls,
const InferredPromotion& on_promotion = {}) {
DataTypePtr target = remove_nullable(target_type);
if (target->equals(*type)) {
return Status::OK();
}
ColumnPtr promoted;
RETURN_IF_ERROR(cast_to_type(target, &promoted));
// Inferred widening is only a storage representation choice. If CAST loses a valid value
// at any array depth, preserve the whole path as JSONB instead. Forced typed-path
// conversion intentionally retains its existing cast-null filtering below.
if (!filter_cast_nulls && target->get_primitive_type() != TYPE_JSONB &&
cast_introduced_null(*column, *promoted)) {
target = jsonb_type();
RETURN_IF_ERROR(
variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()},
make_nullable(target), &promoted));
RETURN_IF_ERROR(cast_to_type(target, &promoted));
DORIS_CHECK(!assert_cast<const ColumnNullable&>(*promoted).has_null());
}
if (on_promotion) {
const DataTypePtr proposed = target;
RETURN_IF_ERROR(on_promotion(&target));
target = remove_nullable(target);
if (!target->equals(*proposed)) {
// For example, the active chunk's 0.1 fits scale 38, but a sealed large Decimal
// does not. Recast the original active values to JSONB, preserving 0.1's scale.
DORIS_CHECK_EQ(target->get_primitive_type(), TYPE_JSONB);
RETURN_IF_ERROR(cast_to_type(target, &promoted));
}
}
const auto& nullable = assert_cast<const ColumnNullable&>(*promoted);
if (nullable.has_null()) {
DORIS_CHECK(filter_cast_nulls);
Expand Down Expand Up @@ -1160,7 +1184,8 @@ VariantPathBuilder::~VariantPathBuilder() = default;
VariantPathBuilder::VariantPathBuilder(VariantPathBuilder&&) noexcept = default;
VariantPathBuilder& VariantPathBuilder::operator=(VariantPathBuilder&&) noexcept = default;

Status VariantPathBuilder::append(VariantRef value, size_t row) {
Status VariantPathBuilder::append(VariantRef value, size_t row,
const InferredPromotion& on_promotion) {
try {
const ScalarPhysical physical = scalar_physical(value);
if (physical.kind == ScalarPhysicalKind::NULL_VALUE) {
Expand Down Expand Up @@ -1189,14 +1214,14 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) {
DataTypePtr common_type = path_least_common_type(_impl->type, inferred_type);
if (_impl->type.get() != common_type.get() &&
!_impl->type->equals(*common_type)) {
RETURN_IF_ERROR(_impl->promote(common_type, false));
RETURN_IF_ERROR(_impl->promote(common_type, false, on_promotion));
}
}
}
}
const bool is_array = !stable_scalar && value_kind(value) == ValueKind::ARRAY;
if (is_array && !value_is_representable(value, _impl->type)) {
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
}

try {
Expand All @@ -1207,7 +1232,7 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) {
if (is_array || _impl->type->get_primitive_type() == TYPE_JSONB) {
throw;
}
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
auto& nullable = assert_cast<ColumnNullable&>(*_impl->column);
append_value(value, _impl->type, &nullable.get_nested_column());
nullable.get_null_map_data().push_back(0);
Expand All @@ -1232,11 +1257,25 @@ Status VariantPathBuilder::complete_rows(size_t rows) {
return Status::OK();
}

Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type) {
Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls) {
if (!_impl->column) {
return _impl->initialize(storage_type);
}
return _impl->promote(storage_type, true);
return _impl->promote(storage_type, filter_cast_nulls);
}

Status VariantPathBuilder::check_inferred_promotion(const DataTypePtr& target_type,
bool* needs_jsonb) const {
const DataTypePtr target = remove_nullable(target_type);
*needs_jsonb = false;
if (!_impl->column || target->equals(*_impl->type) ||
target->get_primitive_type() == TYPE_JSONB) {
return Status::OK();
}
ColumnPtr promoted;
RETURN_IF_ERROR(_impl->cast_to_type(target, &promoted));
*needs_jsonb = cast_introduced_null(*_impl->column, *promoted);
return Status::OK();
}

const PathInData& VariantPathBuilder::path() const {
Expand Down
16 changes: 14 additions & 2 deletions be/src/storage/segment/variant/v2/variant_path_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <span>

Expand Down Expand Up @@ -54,6 +55,10 @@ bool variant_path_type_contains_nothing(const DataTypePtr& type);
// returns.
class VariantPathBuilder final {
public:
// Resolve inferred widening against values held outside this builder, before either side
// commits its cast. The resolver may replace the proposed type with JSONB.
using InferredPromotion = std::function<Status(DataTypePtr*)>;

explicit VariantPathBuilder(PathInData path, size_t prefix_rows = 0);
~VariantPathBuilder();

Expand All @@ -62,9 +67,13 @@ class VariantPathBuilder final {
VariantPathBuilder(const VariantPathBuilder&) = delete;
VariantPathBuilder& operator=(const VariantPathBuilder&) = delete;

Status append(VariantRef value, size_t row);
Status append(VariantRef value, size_t row, const InferredPromotion& on_promotion = {});
Status complete_rows(size_t rows);
Status convert_to(const DataTypePtr& storage_type);
// Inferred conversions preserve values through JSONB fallback; explicit storage coercion
// retains the default cast-null filtering.
Status convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls = true);
// Probe inferred widening without changing values, types or rowids in retained chunks.
Status check_inferred_promotion(const DataTypePtr& target_type, bool* needs_jsonb) const;

const PathInData& path() const;
const DataTypePtr& type() const;
Expand Down Expand Up @@ -107,4 +116,7 @@ VariantPathSelection select_variant_paths(std::span<const VariantPathSelectionCa
size_t max_dynamic_materialized_paths,
bool typed_paths_to_sparse);

// Least common type for a variant path. Used to merge types inferred by independent value chunks.
DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right);

} // namespace doris::segment_v2
Loading
Loading