Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 68 additions & 4 deletions be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@

#include "core/data_type_serde/data_type_variant_v2_serde.h"

#include <arrow/array/array_binary.h>
#include <arrow/array/array_nested.h>
#include <arrow/array/builder_binary.h>
#include <arrow/array/builder_nested.h>
#include <arrow/extension_type.h>
#include <arrow/type.h>

#include <algorithm>
Expand Down Expand Up @@ -47,6 +50,7 @@
#include "core/types.h"
#include "core/value/jsonb_value.h"
#include "core/value/variant/variant_batch_builder.h"
#include "core/value/variant/variant_field.h"
#include "exprs/function/parse/variant_jsonb_parse.h"
#include "exprs/function/parse/variant_string_parse.h"
#include "util/jsonb_document.h"
Expand Down Expand Up @@ -364,10 +368,70 @@ Status DataTypeVariantV2SerDe::read_column_from_pb(IColumn& column, const PValue
return Status::NotSupported("read_column_from_pb with type " + column.get_name());
}

Status DataTypeVariantV2SerDe::read_column_from_arrow(IColumn& column, const arrow::Array*, int64_t,
int64_t, const cctz::time_zone&) const {
return Status::Error(ErrorCode::NOT_IMPLEMENTED_ERROR,
"read_column_from_arrow with type " + column.get_name());
Status DataTypeVariantV2SerDe::read_column_from_arrow(IColumn& column, const arrow::Array* array,
int64_t start, int64_t end,
const cctz::time_zone&) const {
if (array == nullptr) {
return Status::InvalidArgument("Native Variant Arrow array is null");
}
if (start < 0 || end < start || end > array->length()) {
return Status::InvalidArgument("Invalid native Variant Arrow row range [{}, {})", start,
end);
}
if (array->type_id() == arrow::Type::EXTENSION) {
const auto& extension = static_cast<const arrow::ExtensionArray&>(*array);
if (extension.extension_type()->extension_name() != "arrow.parquet.variant") {
return Status::InvalidArgument("Expected native Variant Arrow extension, got {}",
array->type()->ToString());
}
array = extension.storage().get();
}
const auto& type = *array->type();
if (type.id() != arrow::Type::STRUCT || type.num_fields() != 2 ||
type.field(0)->name() != "metadata" || type.field(1)->name() != "value" ||
type.field(0)->type()->id() != arrow::Type::BINARY ||
type.field(1)->type()->id() != arrow::Type::BINARY) {
return Status::InvalidArgument("Expected native Variant Arrow storage, got {}",
type.ToString());
}
RETURN_IF_CATCH_EXCEPTION({
// Nested readers pass row ranges. Validate only that slice to avoid repeatedly walking
// an entire child array when each parent contains a small number of Variant values.
const auto slice = array->Slice(start, end - start);
const auto validation = slice->ValidateFull();
if (!validation.ok()) {
return Status::InvalidArgument("Invalid native Variant Arrow array: {}",
validation.ToString());
}
const auto& storage = assert_cast<const arrow::StructArray&>(*slice);
const auto& metadata = assert_cast<const arrow::BinaryArray&>(*storage.field(0));
const auto& values = assert_cast<const arrow::BinaryArray&>(*storage.field(1));
// Validate untrusted driver bytes in a temporary batch before mutating the destination.
// Copy the payloads so the destination remains valid after Arrow releases its buffers.
VariantBatchBuilder decoded;
for (int64_t i = 0; i < slice->length(); ++i) {
auto row = decoded.begin_row();
if (storage.IsNull(i)) {
row.add_null();
} else {
if (metadata.IsNull(i) || values.IsNull(i)) {
return Status::InvalidArgument(
"Native Variant has a NULL metadata/value child at row {}", i);
}
const auto meta = metadata.GetView(i);
const auto value = values.GetView(i);
const auto ref =
(VariantRef {.metadata = {.data = meta.data(), .size = meta.size()},
.value = {value.data(), value.size()}});
validate_variant_metadata(ref.metadata);
validate_variant_payload(ref);
row.add_value(ref);
}
row.finish();
}
assert_cast<ColumnVariantV2&>(column).insert_encoded_batch(decoded.finish_batch());
});
return Status::OK();
}

namespace {
Expand Down
1 change: 1 addition & 0 deletions be/src/exec/operator/file_scan_operator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
case TFileFormatType::FORMAT_PROTO:
case TFileFormatType::FORMAT_TEXT:
case TFileFormatType::FORMAT_WAL:
case TFileFormatType::FORMAT_ARROW:
return true;
default:
return false;
Expand Down
5 changes: 4 additions & 1 deletion be/src/exec/scan/file_scanner_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,10 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile
} else if (format_type == TFileFormatType::FORMAT_ES_HTTP) {
return table_format_name(range) == "es";
} else if (format_type == TFileFormatType::FORMAT_ARROW) {
return is_supported_arrow_table_format(range);
// A plain Arrow range is an IPC load pipe; query Arrow ranges retain their connector.
return is_supported_arrow_table_format(range) ||
(table_format_name(range) == "NotSet" && params.__isset.src_tuple_id &&
params.__isset.file_type && params.file_type == TFileType::FILE_STREAM);
} else if (format_type == TFileFormatType::FORMAT_LANCE) {
return is_supported_lance_table_format(range);
} else if (format_type == TFileFormatType::FORMAT_JNI) {
Expand Down
10 changes: 10 additions & 0 deletions be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,16 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c
commit_data->__set_file_size(_file_format_transformer->written_len());
commit_data->__set_file_content(TFileContent::DATA);
commit_data->__set_partition_values(_partition_values);
// Split offsets describe the physical file, even when the table disables all column metrics.
if (_file_format_type == TFileFormatType::FORMAT_PARQUET) {
std::vector<int64_t> offsets;
RETURN_IF_ERROR(static_cast<VIcebergParquetWriter*>(_file_format_transformer.get())
->collect_split_offsets_after_close(&offsets));
// Empty files have no splits; leave the optional field absent, as with an older BE.
if (!offsets.empty()) {
commit_data->__set_split_offsets(offsets);
}
}
// ORC collection reopens the file, so honor the FE policy before any footer work.
if (!_collect_column_stats) {
return Status::OK();
Expand Down
36 changes: 36 additions & 0 deletions be/src/exprs/vtopn_pred.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,42 @@ bool topn_string_matches(int comparison, RawComparisonOp op) {

} // namespace

bool VTopNPred::can_evaluate_zonemap_filter() const {
// Capability must remain available before the first bound; readers cache it during open.
return _predicate != nullptr && _children.size() == 1 && _children[0]->is_slot_ref();
}

ZoneMapFilterResult VTopNPred::evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const {
if (!can_evaluate_zonemap_filter()) {
return unsupported_zonemap_filter(ctx);
}
const Field bound = _predicate->get_value();
if (bound.is_null()) {
return ZoneMapFilterResult::kMayMatch;
}
const auto* slot = assert_cast<const VSlotRef*>(_children[0].get());
const auto zone = ctx.zone_map(slot->column_id());
const auto type = ctx.data_type(slot->column_id());
if (zone == nullptr || !expr_zonemap::data_types_compatible(type, slot->data_type())) {
return unsupported_zonemap_filter(ctx);
}
if (_predicate->nulls_first() && zone->has_null) {
return ZoneMapFilterResult::kMayMatch;
}
if (!zone->has_not_null) {
return ZoneMapFilterResult::kNoMatch;
}
if (ctx.floating_nan_count_unknown(slot->column_id()) || bound.is_nan() ||
!expr_zonemap::range_stats_usable_for_zonemap(*zone, type) ||
!expr_zonemap::field_types_compatible(bound.get_type(),
remove_nullable(type)->get_primitive_type())) {
return unsupported_zonemap_filter(ctx);
}
// Keep equality: with ORDER BY a, b a row tied on a can still improve b.
const bool excluded = _predicate->is_asc() ? bound < zone->min_value : zone->max_value < bound;
return excluded ? ZoneMapFilterResult::kNoMatch : ZoneMapFilterResult::kMayMatch;
}

bool VTopNPred::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, int column_id) const {
if (_predicate == nullptr || data_type == nullptr || _children.size() != 1) {
return false;
Expand Down
6 changes: 6 additions & 0 deletions be/src/exprs/vtopn_pred.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ class VTopNPred : public VExpr {
}

int source_node_id() const { return _source_node_id; }
bool is_asc() const { return _predicate->is_asc(); }
bool nulls_first() const { return _predicate->nulls_first(); }

bool can_evaluate_zonemap_filter() const override;
ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override;

Status clone_node(VExprSPtr* cloned_expr) const override {
DORIS_CHECK(cloned_expr != nullptr);
*cloned_expr = VTopNPred::create_shared(clone_texpr_node(), _source_node_id, nullptr);
Expand Down
Loading
Loading