Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions be/src/exec/sink/viceberg_merge_sink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,9 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
_delete_row_count += delete_rows;
_insert_row_count += insert_rows;

// A delete-only plan deliberately omits the data writer so Variant target schemas never enter
// the unsupported Iceberg data-write path. Reject a mismatched FE plan before dereferencing it.
// A delete-only plan omits the data writer, which it does not need and which a BE before Variant
// writes could not open for a Variant target schema. Reject a mismatched FE plan before
// dereferencing it.
if (has_insert && !_writes_data_files) {
return Status::InternalError(
"Iceberg delete-only merge sink received a data insert operation");
Expand Down
11 changes: 10 additions & 1 deletion be/src/format/table/iceberg/arrow_schema_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "format/table/iceberg/arrow_schema_util.h"

#include <arrow/extension/parquet_variant.h>
#include <arrow/extension/uuid.h>
#include <arrow/type.h>
#include <arrow/util/key_value_metadata.h>
Expand All @@ -29,7 +30,8 @@ const char* IcebergArrowSchemaConvertor::MAP_TYPE_VALUE = "mapType";
const char* IcebergArrowSchemaConvertor::UUID_TYPE_VALUE = "uuid";

Status IcebergArrowSchemaConvertor::get_arrow_schema(std::shared_ptr<arrow::Schema>* result) const {
// Field IDs, UUID annotations and timestamp bindings come from the pinned table schema.
// Field IDs, UUID annotations, Variant storage and timestamp bindings come from the pinned
// table schema.
std::vector<std::shared_ptr<arrow::Field>> fields;
RETURN_IF_ERROR(convert_fields(fields));
auto schema = arrow::schema(std::move(fields));
Expand Down Expand Up @@ -108,6 +110,13 @@ Status IcebergArrowSchemaConvertor::convert_to_arrow_field(
arrow_type = arrow::binary();
break;

case iceberg::TypeID::VARIANT:
arrow_type = arrow::extension::variant(arrow::struct_({

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Compact shared dictionaries before serializing VARIANT rows

This newly enabled Arrow mapping reaches DataTypeVariantV2SerDe::write_column_to_iceberg_arrow, which calls write_parquet_variant_arrow(..., compact_metadata=false) and appends each row's entire batch-shared dictionary. For an unpartitioned v3 Parquet target, with batch_size=4096 and one pipeline task, a batch of 4,096 values from PARSE_TO_VARIANT(CONCAT('{"', REPEAT('x',240), CAST(number AS STRING), '":1}')) has only about 1 MiB of JSON but repeats its dictionary into about 4 GB of Arrow metadata. VParquetWriter::write converts the whole block before row-group handling, so it hits the cumulative 2 GiB limit in Arrow 24 BinaryBuilder and fails a valid insert (or exhausts the query memory first). Compact unused dictionary keys per row before this path writes them; the existing compact_metadata=true branch provides that machinery.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed. PARSE_TO_VARIANT encodes a block with one encoder, so its rows share one dictionary, and write_column_to_iceberg_arrow passes compact_metadata=false, so every row appends the whole dictionary. With many distinct keys in one batch, the metadata column passes the 2 GiB limit of Arrow's BinaryBuilder and the INSERT fails; it fails rather than writing anything wrong. That serde came with #68883, and Paimon's write_binary_variant_arrow appends the same shared dictionary for every row on master today. The fix therefore belongs in the shared serde, for both table writers, rather than in this PR's schema mapping. We will fix it in a separate PR for Iceberg and Paimon: compact the dictionary of a row that does not use all of its keys, as the compact_metadata=true branch already does for Flight, with a BE test that bounds the metadata bytes of a high-cardinality batch.

arrow::field("metadata", arrow::binary(), false),
arrow::field("value", arrow::binary(), false),
}));
break;

case iceberg::TypeID::UUID:
metadata[ORIGINAL_TYPE] = UUID_TYPE_VALUE;
arrow_type = _use_uuid_extension ? arrow::extension::uuid() : arrow::fixed_size_binary(16);
Expand Down
2 changes: 2 additions & 0 deletions be/src/format/table/iceberg/types.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ std::unique_ptr<PrimitiveType> Types::from_primitive_string(const std::string& t
return std::make_unique<UUIDType>();
} else if (lower_type_string == "binary") {
return std::make_unique<BinaryType>();
} else if (lower_type_string == "variant") {
return std::make_unique<VariantType>();
} else {
std::regex fixed(R"(fixed\[\s*(\d+)\s*\])");
std::regex decimal(R"(decimal\(\s*(\d+)\s*,\s*(\d+)\s*\))");
Expand Down
10 changes: 10 additions & 0 deletions be/src/format/table/iceberg/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ enum TypeID {
UUID,
FIXED,
BINARY,
VARIANT,
DECIMAL,
STRUCT,
LIST,
Expand Down Expand Up @@ -279,6 +280,15 @@ class BinaryType : public PrimitiveType {
std::string to_string() const override { return "binary"; }
};

class VariantType : public PrimitiveType {
public:
~VariantType() override = default;

TypeID type_id() const override { return TypeID::VARIANT; }

std::string to_string() const override { return "variant"; }
};

class FixedType : public PrimitiveType {
public:
~FixedType() override = default;
Expand Down
47 changes: 44 additions & 3 deletions be/src/format/transformer/viceberg_parquet_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
#include <parquet/schema.h>

#include <algorithm>
#include <optional>
#include <unordered_set>

#include "format/table/iceberg/iceberg_arrow_block_convertor.h"
#include "format/table/parquet_utils.h"
Expand Down Expand Up @@ -51,6 +53,33 @@ static std::string encode_iceberg_bound(const ::parquet::Statistics& stats, std:
encoded.erase(0, start);
return encoded;
}

namespace {

// The Iceberg field whose metrics a Parquet leaf column feeds.
struct IcebergMetricColumn {
int field_id;
bool is_variant_metadata;
};

// The leaves of a Variant carry no Iceberg field id. Its metadata leaf counts the values and nulls
// of the Variant field, and its value leaf feeds no metric: its byte statistics are no logical
// Variant statistics.
std::optional<IcebergMetricColumn> iceberg_metric_column(const ::parquet::schema::Node& leaf) {
const auto* parent = leaf.parent();
const bool is_variant_leaf = parent != nullptr && parent->logical_type() != nullptr &&
parent->logical_type()->is_variant();
if (is_variant_leaf && leaf.name() != "metadata") {
return std::nullopt;
}
const int field_id = is_variant_leaf ? parent->field_id() : leaf.field_id();
// Every field of the Iceberg schema the file is written with carries its id.
DORIS_CHECK_GE(field_id, 0);
return IcebergMetricColumn {field_id, is_variant_leaf};
}

} // namespace

VIcebergParquetWriter::VIcebergParquetWriter(RuntimeState* state, io::FileWriter* file_writer,
const VExprContextSPtrs& output_vexpr_ctxs,
std::vector<std::string> column_names,
Expand Down Expand Up @@ -97,16 +126,28 @@ Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumn
std::map<int, std::string> lower_bounds;
std::map<int, std::string> upper_bounds;
std::map<int, std::shared_ptr<::parquet::Statistics>> merged_column_stats;
// Variant fields get value and null counts, but no column size and no bounds.
std::unordered_set<int> variant_field_ids;

const int num_row_groups = file_metadata->num_row_groups();
const int num_columns = file_metadata->num_columns();
for (int col_idx = 0; col_idx < num_columns; ++col_idx) {
auto field_id = file_metadata->schema()->Column(col_idx)->schema_node()->field_id();
const auto metric_column =
iceberg_metric_column(*file_metadata->schema()->Column(col_idx)->schema_node());
if (!metric_column.has_value()) {
continue;
}
const auto [field_id, is_variant_metadata] = *metric_column;
if (is_variant_metadata) {
variant_field_ids.insert(field_id);
}

for (int rg_idx = 0; rg_idx < num_row_groups; ++rg_idx) {
auto row_group = file_metadata->RowGroup(rg_idx);
auto column_chunk = row_group->ColumnChunk(col_idx);
column_sizes[field_id] += column_chunk->total_compressed_size();
if (!is_variant_metadata) {
column_sizes[field_id] += column_chunk->total_compressed_size();
}

if (column_chunk->is_stats_set()) {
auto column_stat = column_chunk->statistics();
Expand All @@ -129,7 +170,7 @@ Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumn
null_value_counts[field_id] = null_count;
value_counts[field_id] += null_count;
}
if (column_stat->HasMinMax()) {
if (!variant_field_ids.contains(field_id) && column_stat->HasMinMax()) {
has_any_min_max = true;
lower_bounds[field_id] = encode_iceberg_bound(*column_stat, column_stat->EncodeMin());
upper_bounds[field_id] = encode_iceberg_bound(*column_stat, column_stat->EncodeMax());
Expand Down
8 changes: 8 additions & 0 deletions be/src/format_v2/table/iceberg_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,10 @@ static Status build_v2_json_scalar_default(const format::ColumnDefinition& field
const rapidjson::Value& json_value,
std::deque<std::string>* binary_storage, Field* result) {
const auto primitive_type = value_type->get_primitive_type();
if (primitive_type == TYPE_VARIANT) {
return Status::InvalidArgument(
"Iceberg VARIANT initial-default for field '{}' must be NULL", field.name);
}
std::string serialized_value = iceberg_json_scalar_text(json_value);
const bool binary_like =
field.initial_default_value_is_base64 || primitive_type == TYPE_VARBINARY;
Expand Down Expand Up @@ -606,6 +610,10 @@ static Status build_v2_initial_default_field(const format::ColumnDefinition& fie

const auto value_type = remove_nullable(data_type);
const auto primitive_type = value_type->get_primitive_type();
if (primitive_type == TYPE_VARIANT) {
return Status::InvalidArgument(
"Iceberg VARIANT initial-default for field '{}' must be NULL", field.name);
}
if (is_complex_type(primitive_type)) {
rapidjson::Document document;
document.Parse(field.initial_default_value->data(), field.initial_default_value->size());
Expand Down
8 changes: 6 additions & 2 deletions be/test/format/arrow/arrow_block_convertor_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "format/arrow/arrow_block_convertor.h"

#include <arrow/api.h>
#include <arrow/extension_type.h>
#include <arrow/io/api.h>
#include <arrow/ipc/api.h>
#include <gtest/gtest.h>
Expand Down Expand Up @@ -79,13 +80,16 @@ TEST_F(ArrowBlockConvertorTest, ParquetRejectsMismatchedColumnNames) {

TEST_F(ArrowBlockConvertorTest, IcebergBuildsItsOwnSchemaAndMetadata) {
const std::string json =
R"({"type":"struct","fields":[{"id":7,"name":"payload","required":false,"type":"string"}]})";
R"({"type":"struct","fields":[{"id":7,"name":"payload","required":false,"type":"variant"}]})";
auto schema = iceberg::SchemaParser::from_json(json);
iceberg::IcebergArrowBlockConvertor converter(*schema, &json, "UTC", cctz::utc_time_zone());
ASSERT_TRUE(converter.init().ok());
const auto& arrow_schema = converter.arrow_schema();
ASSERT_NE(nullptr, arrow_schema);
EXPECT_EQ(arrow::Type::STRING, arrow_schema->field(0)->type()->id());
ASSERT_EQ(arrow::Type::EXTENSION, arrow_schema->field(0)->type()->id());
const auto& variant = static_cast<const arrow::ExtensionType&>(*arrow_schema->field(0)->type());
EXPECT_EQ(arrow::Type::STRUCT, variant.storage_type()->id());
EXPECT_EQ("arrow.parquet.variant", variant.extension_name());
ASSERT_NE(nullptr, arrow_schema->metadata());
EXPECT_EQ(json, arrow_schema->metadata()->Get("iceberg.schema").ValueOrDie());
EXPECT_EQ("7", arrow_schema->field(0)->metadata()->Get("PARQUET:field_id").ValueOrDie());
Expand Down
97 changes: 97 additions & 0 deletions be/test/format/table/iceberg/arrow_schema_util_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "format/table/iceberg/arrow_schema_util.h"

#include <arrow/api.h>
#include <arrow/extension/parquet_variant.h>
#include <arrow/io/api.h>
#include <arrow/status.h>
#include <arrow/type.h>
Expand All @@ -28,6 +29,7 @@
#include <parquet/arrow/writer.h>
#include <parquet/schema.h>

#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/column/column_array.h"
#include "core/column/column_nullable.h"
Expand Down Expand Up @@ -358,6 +360,101 @@ TEST(ArrowSchemaUtilTest, test_binary_field_types) {
EXPECT_EQ("uuid", fields[3]->metadata()->Get("originalType").ValueUnsafe());
}

TEST(ArrowSchemaUtilTest, test_variant_field) {
std::vector<NestedField> nested_fields;
nested_fields.emplace_back(true, 21, "payload", std::make_unique<VariantType>(), std::nullopt);
Schema schema(1, std::move(nested_fields));

std::vector<std::shared_ptr<arrow::Field>> fields;
Status st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields);
ASSERT_TRUE(st.ok()) << st;
ASSERT_EQ(1, fields.size());
EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe());
ASSERT_EQ(arrow::Type::EXTENSION, fields[0]->type()->id());

const auto& extension = static_cast<const arrow::ExtensionType&>(*fields[0]->type());
EXPECT_EQ("arrow.parquet.variant", extension.extension_name());
ASSERT_EQ(arrow::Type::STRUCT, extension.storage_type()->id());
ASSERT_EQ(2, extension.storage_type()->num_fields());
EXPECT_EQ("metadata", extension.storage_type()->field(0)->name());
EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(0)->type()->id());
EXPECT_FALSE(extension.storage_type()->field(0)->nullable());
EXPECT_FALSE(extension.storage_type()->field(0)->HasMetadata());
EXPECT_EQ("value", extension.storage_type()->field(1)->name());
EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(1)->type()->id());
EXPECT_FALSE(extension.storage_type()->field(1)->nullable());
EXPECT_FALSE(extension.storage_type()->field(1)->HasMetadata());
}

TEST(ArrowSchemaUtilTest, test_nested_variant_fields) {
const std::string schema_json = R"({
"type": "struct",
"fields": [
{
"id": 10,
"name": "info",
"required": false,
"type": {
"type": "struct",
"fields": [
{"id": 11, "name": "payload", "required": false, "type": "variant"}
]
}
},
{
"id": 20,
"name": "events",
"required": false,
"type": {
"type": "list",
"element-id": 21,
"element": "variant",
"element-required": false
}
},
{
"id": 30,
"name": "attrs",
"required": false,
"type": {
"type": "map",
"key-id": 31,
"key": "string",
"value-id": 32,
"value": "variant",
"value-required": false
}
}
]
})";
std::unique_ptr<Schema> schema = SchemaParser::from_json(schema_json);

std::vector<std::shared_ptr<arrow::Field>> fields;
Status status = IcebergArrowSchemaConvertor(*schema, "UTC").convert_fields(fields);
ASSERT_TRUE(status.ok()) << status;
ASSERT_EQ(3, fields.size());

const auto& info = assert_cast<const arrow::StructType&>(*fields[0]->type());
ASSERT_EQ(arrow::Type::EXTENSION, info.field(0)->type()->id());
EXPECT_EQ("11", info.field(0)->metadata()->Get(pfid).ValueUnsafe());

const auto& events = assert_cast<const arrow::ListType&>(*fields[1]->type());
ASSERT_EQ(arrow::Type::EXTENSION, events.value_type()->id());
EXPECT_EQ("21", events.value_field()->metadata()->Get(pfid).ValueUnsafe());

const auto& attrs = assert_cast<const arrow::MapType&>(*fields[2]->type());
ASSERT_EQ(arrow::Type::EXTENSION, attrs.item_type()->id());
EXPECT_EQ("32", attrs.item_field()->metadata()->Get(pfid).ValueUnsafe());

for (const auto& variant_type :
{info.field(0)->type(), events.value_type(), attrs.item_type()}) {
const auto& extension = static_cast<const arrow::ExtensionType&>(*variant_type);
EXPECT_EQ("arrow.parquet.variant", extension.extension_name());
EXPECT_EQ("struct<metadata: binary not null, value: binary not null>",
extension.storage_type()->ToString());
}
}

TEST(ArrowSchemaUtilTest, test_parquet_filed_id) {
std::string test_dir = "ut_dir/test_parquet_filed_id";
Status st;
Expand Down
19 changes: 19 additions & 0 deletions be/test/format/table/iceberg/schema_parser_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ const std::string valid_struct_json = R"({
]
})";

const std::string variant_struct_json = R"({
"type": "struct",
"fields": [
{
"id": 30,
"name": "payload",
"type": "variant",
"required": false
}
]
})";

const std::string invalid_json = R"({
"type": "struct",
"fields": [
Expand Down Expand Up @@ -202,6 +214,13 @@ TEST(SchemaParserTest, parse_valid_struct) {
EXPECT_EQ(age_field->field_type()->as_primitive_type()->to_string(), "int");
}

TEST(SchemaParserTest, parse_variant) {
std::unique_ptr<Schema> schema = SchemaParser::from_json(variant_struct_json);
ASSERT_NE(schema, nullptr);
ASSERT_NE(schema->find_field(30), nullptr);
EXPECT_EQ(schema->find_field(30)->field_type()->type_id(), TypeID::VARIANT);
}

//TEST(SchemaParserTest, parse_invalid_json) {
// EXPECT_THROW(SchemaParser::from_json(invalid_json), doris::Exception);
//}
Expand Down
7 changes: 7 additions & 0 deletions be/test/format/table/iceberg/type_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ TEST(BinaryTypeTest, to_string) {
EXPECT_EQ(binary_type.type_id(), TypeID::BINARY);
}

TEST(VariantTypeTest, to_string) {
VariantType variant_type;
EXPECT_EQ(variant_type.to_string(), "variant");
EXPECT_EQ(variant_type.type_id(), TypeID::VARIANT);
}

TEST(FixedTypeTest, to_string) {
FixedType fixed_type(16);
EXPECT_EQ(fixed_type.to_string(), "fixed[16]");
Expand Down Expand Up @@ -146,6 +152,7 @@ TEST(PrimitiveTypesTest, from_primitive_string) {
EXPECT_EQ(Types::from_primitive_string("string")->type_id(), TypeID::STRING);
EXPECT_EQ(Types::from_primitive_string("uuid")->type_id(), TypeID::UUID);
EXPECT_EQ(Types::from_primitive_string("binary")->type_id(), TypeID::BINARY);
EXPECT_EQ(Types::from_primitive_string("variant")->type_id(), TypeID::VARIANT);
EXPECT_EQ(Types::from_primitive_string("fixed[16]")->type_id(), TypeID::FIXED);
EXPECT_EQ(Types::from_primitive_string("decimal(10, 2)")->type_id(), TypeID::DECIMAL);
}
Expand Down
Loading
Loading