From ed28e47268ee88ae96157caafe23d7c1fbebaa0b Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Oct 2026 10:09:41 +0100 Subject: [PATCH 1/4] [feature](iceberg) Write Iceberg VARIANT columns and require NULL VARIANT defaults ### What problem does this PR solve? Issue Number: None Related PR: #66620, #67323 Problem Summary: Port branch-4.1 #66620 (Iceberg Variant V2 writes) and #67323 (NULL defaults for VARIANT) to master. Master read Iceberg VARIANT columns but rejected every write to a table holding one ("Iceberg VARIANT columns are read-only and cannot be written") and every VARIANT in Iceberg DDL ("Unsupported type for Iceberg: VARIANT"), and BE could neither parse "variant" in the Iceberg schema JSON nor map it to Arrow. A connector write whose source mixed a Variant with a number in UNION, IF, CASE, a recursive CTE or LATERAL VIEW got a DECIMAL common type, so object and array Variants were silently written as NULL (Paimon today). BE: the Iceberg type system and Arrow schema handle VARIANT (arrow.parquet.variant over struct); the Iceberg Parquet writer counts the values and nulls of a Variant on its metadata leaf, under the Variant's field id, and records no size or bounds for it; the format v2 Iceberg reader rejects a non-NULL VARIANT initial default. FE: the Iceberg connector maps a top-level VARIANT in CREATE TABLE, CTAS and ADD COLUMN and rejects a nested one; requires format version 3 and Parquet data files for them and for writes of data files, and rejects write.parquet.shred-variants; keeps MODIFY of a VARIANT column to its comment, nullability and position, and rejects type changes to or from VARIANT; collects column metrics for Variant fields; requires NULL VARIANT defaults. VariantWritePlanValidator rejects an implicit Variant-to-non- Variant cast in the lineage of a Variant target column of any connector sink: INSERT, INSERT OVERWRITE, CTAS, rewrite, Paimon UPDATE / MERGE and Iceberg UPDATE / MERGE, whose merge sink now casts its output columns by position, so a data column named `operation` is no longer left uncast. Unlike branch-4.x, the validator does not inspect the condition of an IF or a CASE branch, which only chooses the written value, its message names no engine, and it also covers Paimon UPDATE / MERGE. ### Release note Doris can write VARIANT columns of Iceberg format-version 3 tables with Parquet data files (INSERT, INSERT OVERWRITE, UPDATE, MERGE and rewrite_data_files) and create or add top-level VARIANT columns. A write into a VARIANT column of an Iceberg or Paimon table now fails when its source implicitly casts a VARIANT to another type, for example IF(cond, payload, 1) or a UNION of a Variant and a number, instead of storing object and array values as NULL; write an explicit CAST to keep such a conversion. ### Check List (For Author) - Test: Regression test / Unit Test - BE UT: Iceberg Variant type, schema parsing, Arrow schema, Parquet writer metrics and NULL initial defaults (474 cases of the related suites pass under ASAN). - FE UT: VariantWritePlanValidatorTest and the merge sink binding in fe-core (131 cases of 12 classes), and all 1580 cases of fe-connector-iceberg. - Regression: test_iceberg_variant_read, the new test_iceberg_variant_defaults and test_paimon_write_variant_errors; all Paimon and Iceberg write suites and the Iceberg row-lineage DML suites pass locally, except test_paimon_write_thread_lifecycle, which needs Linux /proc. - Behavior changed: Yes (Iceberg VARIANT columns become writable; lossy implicit casts into VARIANT targets are rejected) - Does this need documentation: Yes (Iceberg VARIANT writes and DDL) Co-authored-by: daidai Co-Authored-By: Claude Opus 5.5 (1M context) --- be/src/exec/sink/viceberg_merge_sink.cpp | 5 +- .../table/iceberg/arrow_schema_util.cpp | 11 +- be/src/format/table/iceberg/types.cpp | 2 + be/src/format/table/iceberg/types.h | 10 + .../transformer/viceberg_parquet_writer.cpp | 47 +- be/src/format_v2/table/iceberg_reader.cpp | 8 + .../arrow/arrow_block_convertor_test.cpp | 8 +- .../table/iceberg/arrow_schema_util_test.cpp | 97 +++ .../table/iceberg/schema_parser_test.cpp | 19 + be/test/format/table/iceberg/type_test.cpp | 7 + .../transformer/vparquet_writer_test.cpp | 161 +++++ .../format_v2/table/iceberg_reader_test.cpp | 102 +++ .../connector/iceberg/IcebergCatalogOps.java | 20 +- .../iceberg/IcebergConnectorMetadata.java | 9 +- .../iceberg/IcebergSchemaBuilder.java | 58 +- .../connector/iceberg/IcebergSchemaUtils.java | 3 + .../iceberg/IcebergWritePlanProvider.java | 37 +- .../iceberg/IcebergWriteSchemaContext.java | 2 + .../iceberg/IcebergWriterHelper.java | 34 +- ...dIcebergCatalogOpsColumnEvolutionTest.java | 76 ++ .../IcebergConnectorMetadataDdlTest.java | 47 ++ .../iceberg/IcebergSchemaBuilderTest.java | 68 ++ .../iceberg/IcebergSchemaUtilsTest.java | 10 + .../iceberg/IcebergWritePlanProviderTest.java | 147 +++- .../datasource/VariantWritePlanValidator.java | 448 ++++++++++++ .../rules/analysis/BindExpression.java | 116 +-- .../nereids/rules/analysis/BindSink.java | 8 + .../doris/nereids/types/VariantType.java | 16 + .../VariantWritePlanValidatorTest.java | 426 +++++++++++ .../rules/analysis/BindExpressionTest.java | 65 ++ .../ExternalRowLevelMergePlanBuilderTest.java | 22 + .../iceberg/test_iceberg_variant_defaults.out | 23 + .../iceberg/test_iceberg_variant_read.out | 7 + .../test_paimon_write_variant_errors.out | 6 + .../test_iceberg_variant_defaults.groovy | 156 ++++ .../iceberg/test_iceberg_variant_read.groovy | 676 +++++++++++++++++- .../test_paimon_write_variant_errors.groovy | 197 +++-- 37 files changed, 2951 insertions(+), 203 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp index b025ce2cb83284..4548f0449b6eed 100644 --- a/be/src/exec/sink/viceberg_merge_sink.cpp +++ b/be/src/exec/sink/viceberg_merge_sink.cpp @@ -168,8 +168,9 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) { _delete_row_count += delete_rows; _insert_row_count += insert_rows; - // A delete-only plan deliberately omits the data writer so Variant target schemas never enter - // the unsupported Iceberg data-write path. Reject a mismatched FE plan before dereferencing it. + // A delete-only plan omits the data writer, which it does not need and which a BE before Variant + // writes could not open for a Variant target schema. Reject a mismatched FE plan before + // dereferencing it. if (has_insert && !_writes_data_files) { return Status::InternalError( "Iceberg delete-only merge sink received a data insert operation"); diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index 18aec26888f8e3..316549c4a663fb 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -17,6 +17,7 @@ #include "format/table/iceberg/arrow_schema_util.h" +#include #include #include #include @@ -29,7 +30,8 @@ const char* IcebergArrowSchemaConvertor::MAP_TYPE_VALUE = "mapType"; const char* IcebergArrowSchemaConvertor::UUID_TYPE_VALUE = "uuid"; Status IcebergArrowSchemaConvertor::get_arrow_schema(std::shared_ptr* result) const { - // Field IDs, UUID annotations and timestamp bindings come from the pinned table schema. + // Field IDs, UUID annotations, Variant storage and timestamp bindings come from the pinned + // table schema. std::vector> fields; RETURN_IF_ERROR(convert_fields(fields)); auto schema = arrow::schema(std::move(fields)); @@ -108,6 +110,13 @@ Status IcebergArrowSchemaConvertor::convert_to_arrow_field( arrow_type = arrow::binary(); break; + case iceberg::TypeID::VARIANT: + arrow_type = arrow::extension::variant(arrow::struct_({ + arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), false), + })); + break; + case iceberg::TypeID::UUID: metadata[ORIGINAL_TYPE] = UUID_TYPE_VALUE; arrow_type = _use_uuid_extension ? arrow::extension::uuid() : arrow::fixed_size_binary(16); diff --git a/be/src/format/table/iceberg/types.cpp b/be/src/format/table/iceberg/types.cpp index 7046e3b4f6fe58..8a40a716b933fb 100644 --- a/be/src/format/table/iceberg/types.cpp +++ b/be/src/format/table/iceberg/types.cpp @@ -174,6 +174,8 @@ std::unique_ptr Types::from_primitive_string(const std::string& t return std::make_unique(); } else if (lower_type_string == "binary") { return std::make_unique(); + } else if (lower_type_string == "variant") { + return std::make_unique(); } else { std::regex fixed(R"(fixed\[\s*(\d+)\s*\])"); std::regex decimal(R"(decimal\(\s*(\d+)\s*,\s*(\d+)\s*\))"); diff --git a/be/src/format/table/iceberg/types.h b/be/src/format/table/iceberg/types.h index b4801dd0a5e6f2..9b8fd4dc0f629a 100644 --- a/be/src/format/table/iceberg/types.h +++ b/be/src/format/table/iceberg/types.h @@ -48,6 +48,7 @@ enum TypeID { UUID, FIXED, BINARY, + VARIANT, DECIMAL, STRUCT, LIST, @@ -279,6 +280,15 @@ class BinaryType : public PrimitiveType { std::string to_string() const override { return "binary"; } }; +class VariantType : public PrimitiveType { +public: + ~VariantType() override = default; + + TypeID type_id() const override { return TypeID::VARIANT; } + + std::string to_string() const override { return "variant"; } +}; + class FixedType : public PrimitiveType { public: ~FixedType() override = default; diff --git a/be/src/format/transformer/viceberg_parquet_writer.cpp b/be/src/format/transformer/viceberg_parquet_writer.cpp index ceff37500a5482..5b951ec77b1b6f 100644 --- a/be/src/format/transformer/viceberg_parquet_writer.cpp +++ b/be/src/format/transformer/viceberg_parquet_writer.cpp @@ -21,6 +21,8 @@ #include #include +#include +#include #include "format/table/iceberg/iceberg_arrow_block_convertor.h" #include "format/table/parquet_utils.h" @@ -51,6 +53,33 @@ static std::string encode_iceberg_bound(const ::parquet::Statistics& stats, std: encoded.erase(0, start); return encoded; } + +namespace { + +// The Iceberg field whose metrics a Parquet leaf column feeds. +struct IcebergMetricColumn { + int field_id; + bool is_variant_metadata; +}; + +// The leaves of a Variant carry no Iceberg field id. Its metadata leaf counts the values and nulls +// of the Variant field, and its value leaf feeds no metric: its byte statistics are no logical +// Variant statistics. +std::optional iceberg_metric_column(const ::parquet::schema::Node& leaf) { + const auto* parent = leaf.parent(); + const bool is_variant_leaf = parent != nullptr && parent->logical_type() != nullptr && + parent->logical_type()->is_variant(); + if (is_variant_leaf && leaf.name() != "metadata") { + return std::nullopt; + } + const int field_id = is_variant_leaf ? parent->field_id() : leaf.field_id(); + // Every field of the Iceberg schema the file is written with carries its id. + DORIS_CHECK_GE(field_id, 0); + return IcebergMetricColumn {field_id, is_variant_leaf}; +} + +} // namespace + VIcebergParquetWriter::VIcebergParquetWriter(RuntimeState* state, io::FileWriter* file_writer, const VExprContextSPtrs& output_vexpr_ctxs, std::vector column_names, @@ -97,16 +126,28 @@ Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumn std::map lower_bounds; std::map upper_bounds; std::map> merged_column_stats; + // Variant fields get value and null counts, but no column size and no bounds. + std::unordered_set variant_field_ids; const int num_row_groups = file_metadata->num_row_groups(); const int num_columns = file_metadata->num_columns(); for (int col_idx = 0; col_idx < num_columns; ++col_idx) { - auto field_id = file_metadata->schema()->Column(col_idx)->schema_node()->field_id(); + const auto metric_column = + iceberg_metric_column(*file_metadata->schema()->Column(col_idx)->schema_node()); + if (!metric_column.has_value()) { + continue; + } + const auto [field_id, is_variant_metadata] = *metric_column; + if (is_variant_metadata) { + variant_field_ids.insert(field_id); + } for (int rg_idx = 0; rg_idx < num_row_groups; ++rg_idx) { auto row_group = file_metadata->RowGroup(rg_idx); auto column_chunk = row_group->ColumnChunk(col_idx); - column_sizes[field_id] += column_chunk->total_compressed_size(); + if (!is_variant_metadata) { + column_sizes[field_id] += column_chunk->total_compressed_size(); + } if (column_chunk->is_stats_set()) { auto column_stat = column_chunk->statistics(); @@ -129,7 +170,7 @@ Status VIcebergParquetWriter::collect_file_statistics_after_close(TIcebergColumn null_value_counts[field_id] = null_count; value_counts[field_id] += null_count; } - if (column_stat->HasMinMax()) { + if (!variant_field_ids.contains(field_id) && column_stat->HasMinMax()) { has_any_min_max = true; lower_bounds[field_id] = encode_iceberg_bound(*column_stat, column_stat->EncodeMin()); upper_bounds[field_id] = encode_iceberg_bound(*column_stat, column_stat->EncodeMax()); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 87924be5966489..1068567076b7d9 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -524,6 +524,10 @@ static Status build_v2_json_scalar_default(const format::ColumnDefinition& field const rapidjson::Value& json_value, std::deque* binary_storage, Field* result) { const auto primitive_type = value_type->get_primitive_type(); + if (primitive_type == TYPE_VARIANT) { + return Status::InvalidArgument( + "Iceberg VARIANT initial-default for field '{}' must be NULL", field.name); + } std::string serialized_value = iceberg_json_scalar_text(json_value); const bool binary_like = field.initial_default_value_is_base64 || primitive_type == TYPE_VARBINARY; @@ -606,6 +610,10 @@ static Status build_v2_initial_default_field(const format::ColumnDefinition& fie const auto value_type = remove_nullable(data_type); const auto primitive_type = value_type->get_primitive_type(); + if (primitive_type == TYPE_VARIANT) { + return Status::InvalidArgument( + "Iceberg VARIANT initial-default for field '{}' must be NULL", field.name); + } if (is_complex_type(primitive_type)) { rapidjson::Document document; document.Parse(field.initial_default_value->data(), field.initial_default_value->size()); diff --git a/be/test/format/arrow/arrow_block_convertor_test.cpp b/be/test/format/arrow/arrow_block_convertor_test.cpp index a2c038d774b43f..f3db16d59de4cf 100644 --- a/be/test/format/arrow/arrow_block_convertor_test.cpp +++ b/be/test/format/arrow/arrow_block_convertor_test.cpp @@ -18,6 +18,7 @@ #include "format/arrow/arrow_block_convertor.h" #include +#include #include #include #include @@ -79,13 +80,16 @@ TEST_F(ArrowBlockConvertorTest, ParquetRejectsMismatchedColumnNames) { TEST_F(ArrowBlockConvertorTest, IcebergBuildsItsOwnSchemaAndMetadata) { const std::string json = - R"({"type":"struct","fields":[{"id":7,"name":"payload","required":false,"type":"string"}]})"; + R"({"type":"struct","fields":[{"id":7,"name":"payload","required":false,"type":"variant"}]})"; auto schema = iceberg::SchemaParser::from_json(json); iceberg::IcebergArrowBlockConvertor converter(*schema, &json, "UTC", cctz::utc_time_zone()); ASSERT_TRUE(converter.init().ok()); const auto& arrow_schema = converter.arrow_schema(); ASSERT_NE(nullptr, arrow_schema); - EXPECT_EQ(arrow::Type::STRING, arrow_schema->field(0)->type()->id()); + ASSERT_EQ(arrow::Type::EXTENSION, arrow_schema->field(0)->type()->id()); + const auto& variant = static_cast(*arrow_schema->field(0)->type()); + EXPECT_EQ(arrow::Type::STRUCT, variant.storage_type()->id()); + EXPECT_EQ("arrow.parquet.variant", variant.extension_name()); ASSERT_NE(nullptr, arrow_schema->metadata()); EXPECT_EQ(json, arrow_schema->metadata()->Get("iceberg.schema").ValueOrDie()); EXPECT_EQ("7", arrow_schema->field(0)->metadata()->Get("PARQUET:field_id").ValueOrDie()); diff --git a/be/test/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/format/table/iceberg/arrow_schema_util_test.cpp index da85ac4b764921..9d84b9e766b0d2 100644 --- a/be/test/format/table/iceberg/arrow_schema_util_test.cpp +++ b/be/test/format/table/iceberg/arrow_schema_util_test.cpp @@ -18,6 +18,7 @@ #include "format/table/iceberg/arrow_schema_util.h" #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include +#include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_array.h" #include "core/column/column_nullable.h" @@ -358,6 +360,101 @@ TEST(ArrowSchemaUtilTest, test_binary_field_types) { EXPECT_EQ("uuid", fields[3]->metadata()->Get("originalType").ValueUnsafe()); } +TEST(ArrowSchemaUtilTest, test_variant_field) { + std::vector nested_fields; + nested_fields.emplace_back(true, 21, "payload", std::make_unique(), std::nullopt); + Schema schema(1, std::move(nested_fields)); + + std::vector> fields; + Status st = IcebergArrowSchemaConvertor(schema, "utc").convert_fields(fields); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(1, fields.size()); + EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe()); + ASSERT_EQ(arrow::Type::EXTENSION, fields[0]->type()->id()); + + const auto& extension = static_cast(*fields[0]->type()); + EXPECT_EQ("arrow.parquet.variant", extension.extension_name()); + ASSERT_EQ(arrow::Type::STRUCT, extension.storage_type()->id()); + ASSERT_EQ(2, extension.storage_type()->num_fields()); + EXPECT_EQ("metadata", extension.storage_type()->field(0)->name()); + EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(0)->type()->id()); + EXPECT_FALSE(extension.storage_type()->field(0)->nullable()); + EXPECT_FALSE(extension.storage_type()->field(0)->HasMetadata()); + EXPECT_EQ("value", extension.storage_type()->field(1)->name()); + EXPECT_EQ(arrow::Type::BINARY, extension.storage_type()->field(1)->type()->id()); + EXPECT_FALSE(extension.storage_type()->field(1)->nullable()); + EXPECT_FALSE(extension.storage_type()->field(1)->HasMetadata()); +} + +TEST(ArrowSchemaUtilTest, test_nested_variant_fields) { + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + { + "id": 10, + "name": "info", + "required": false, + "type": { + "type": "struct", + "fields": [ + {"id": 11, "name": "payload", "required": false, "type": "variant"} + ] + } + }, + { + "id": 20, + "name": "events", + "required": false, + "type": { + "type": "list", + "element-id": 21, + "element": "variant", + "element-required": false + } + }, + { + "id": 30, + "name": "attrs", + "required": false, + "type": { + "type": "map", + "key-id": 31, + "key": "string", + "value-id": 32, + "value": "variant", + "value-required": false + } + } + ] + })"; + std::unique_ptr schema = SchemaParser::from_json(schema_json); + + std::vector> fields; + Status status = IcebergArrowSchemaConvertor(*schema, "UTC").convert_fields(fields); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(3, fields.size()); + + const auto& info = assert_cast(*fields[0]->type()); + ASSERT_EQ(arrow::Type::EXTENSION, info.field(0)->type()->id()); + EXPECT_EQ("11", info.field(0)->metadata()->Get(pfid).ValueUnsafe()); + + const auto& events = assert_cast(*fields[1]->type()); + ASSERT_EQ(arrow::Type::EXTENSION, events.value_type()->id()); + EXPECT_EQ("21", events.value_field()->metadata()->Get(pfid).ValueUnsafe()); + + const auto& attrs = assert_cast(*fields[2]->type()); + ASSERT_EQ(arrow::Type::EXTENSION, attrs.item_type()->id()); + EXPECT_EQ("32", attrs.item_field()->metadata()->Get(pfid).ValueUnsafe()); + + for (const auto& variant_type : + {info.field(0)->type(), events.value_type(), attrs.item_type()}) { + const auto& extension = static_cast(*variant_type); + EXPECT_EQ("arrow.parquet.variant", extension.extension_name()); + EXPECT_EQ("struct", + extension.storage_type()->ToString()); + } +} + TEST(ArrowSchemaUtilTest, test_parquet_filed_id) { std::string test_dir = "ut_dir/test_parquet_filed_id"; Status st; diff --git a/be/test/format/table/iceberg/schema_parser_test.cpp b/be/test/format/table/iceberg/schema_parser_test.cpp index 427e0e3c05cbed..f48afd53470cf2 100644 --- a/be/test/format/table/iceberg/schema_parser_test.cpp +++ b/be/test/format/table/iceberg/schema_parser_test.cpp @@ -49,6 +49,18 @@ const std::string valid_struct_json = R"({ ] })"; +const std::string variant_struct_json = R"({ + "type": "struct", + "fields": [ + { + "id": 30, + "name": "payload", + "type": "variant", + "required": false + } + ] +})"; + const std::string invalid_json = R"({ "type": "struct", "fields": [ @@ -202,6 +214,13 @@ TEST(SchemaParserTest, parse_valid_struct) { EXPECT_EQ(age_field->field_type()->as_primitive_type()->to_string(), "int"); } +TEST(SchemaParserTest, parse_variant) { + std::unique_ptr schema = SchemaParser::from_json(variant_struct_json); + ASSERT_NE(schema, nullptr); + ASSERT_NE(schema->find_field(30), nullptr); + EXPECT_EQ(schema->find_field(30)->field_type()->type_id(), TypeID::VARIANT); +} + //TEST(SchemaParserTest, parse_invalid_json) { // EXPECT_THROW(SchemaParser::from_json(invalid_json), doris::Exception); //} diff --git a/be/test/format/table/iceberg/type_test.cpp b/be/test/format/table/iceberg/type_test.cpp index 73bd2fd54df364..8ede6af24b786b 100644 --- a/be/test/format/table/iceberg/type_test.cpp +++ b/be/test/format/table/iceberg/type_test.cpp @@ -40,6 +40,12 @@ TEST(BinaryTypeTest, to_string) { EXPECT_EQ(binary_type.type_id(), TypeID::BINARY); } +TEST(VariantTypeTest, to_string) { + VariantType variant_type; + EXPECT_EQ(variant_type.to_string(), "variant"); + EXPECT_EQ(variant_type.type_id(), TypeID::VARIANT); +} + TEST(FixedTypeTest, to_string) { FixedType fixed_type(16); EXPECT_EQ(fixed_type.to_string(), "fixed[16]"); @@ -146,6 +152,7 @@ TEST(PrimitiveTypesTest, from_primitive_string) { EXPECT_EQ(Types::from_primitive_string("string")->type_id(), TypeID::STRING); EXPECT_EQ(Types::from_primitive_string("uuid")->type_id(), TypeID::UUID); EXPECT_EQ(Types::from_primitive_string("binary")->type_id(), TypeID::BINARY); + EXPECT_EQ(Types::from_primitive_string("variant")->type_id(), TypeID::VARIANT); EXPECT_EQ(Types::from_primitive_string("fixed[16]")->type_id(), TypeID::FIXED); EXPECT_EQ(Types::from_primitive_string("decimal(10, 2)")->type_id(), TypeID::DECIMAL); } diff --git a/be/test/format/transformer/vparquet_writer_test.cpp b/be/test/format/transformer/vparquet_writer_test.cpp index d296f2c586d8b4..c4b05cf045ef32 100644 --- a/be/test/format/transformer/vparquet_writer_test.cpp +++ b/be/test/format/transformer/vparquet_writer_test.cpp @@ -32,14 +32,17 @@ #include "core/column/column_array.h" #include "core/column/column_decimal.h" #include "core/column/column_nullable.h" +#include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_variant_v2.h" #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/vhive_partition_writer.h" +#include "exprs/function/parse/variant_string_parse.h" #include "format/table/iceberg/schema_parser.h" #include "format/transformer/viceberg_parquet_writer.h" #include "io/fs/file_writer.h" @@ -166,6 +169,164 @@ TEST_F(VParquetWriterTest, WritesIcebergPrimitiveAndCollectsMetrics) { EXPECT_EQ(std::string("\x03\x00\x00\x00", 4), stats.upper_bounds.at(7)); } +TEST_F(VParquetWriterTest, WritesIcebergVariantAndCollectsLogicalMetrics) { + auto variant_type = std::make_shared(); + auto nullable_variant_type = make_nullable(variant_type); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {nullable_variant_type}); + + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + {"id": 2, "name": "payload", "required": false, "type": "variant"} + ] + })"; + std::unique_ptr schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + ParquetFileOptions options {.compression_type = TParquetCompressionType::UNCOMPRESSED, + .parquet_version = TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; + VIcebergParquetWriter transformer(&state, file_writer.get(), output_exprs, {"payload"}, false, + options, &schema_json, *schema); + ASSERT_TRUE(transformer.open().ok()); + + JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (std::string_view json : {std::string_view {R"({"name":"doris","n":1})"}, + std::string_view {"null"}, std::string_view {R"([1,2,3])"}}) { + encoder.add_json({json.data(), json.size()}); + } + auto variant_column = ColumnVariantV2::create(); + variant_column->insert_encoded_batch(encoder.finish_batch()); + auto null_map = ColumnUInt8::create(); + null_map->get_data().assign({0, 0, 1}); + auto nullable_column = ColumnNullable::create(std::move(variant_column), std::move(null_map)); + + Block block; + block.insert( + ColumnWithTypeAndName(std::move(nullable_column), nullable_variant_type, "payload")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + TIcebergColumnStats stats; + ASSERT_TRUE(transformer.collect_file_statistics_after_close(&stats).ok()); + EXPECT_EQ(0, stats.column_sizes.count(-1)); + EXPECT_EQ(0, stats.value_counts.count(-1)); + EXPECT_EQ(0, stats.null_value_counts.count(-1)); + EXPECT_EQ(0, stats.column_sizes.count(2)); + ASSERT_EQ(1, stats.value_counts.count(2)); + EXPECT_EQ(3, stats.value_counts.at(2)); + ASSERT_TRUE(stats.__isset.null_value_counts); + ASSERT_EQ(1, stats.null_value_counts.count(2)); + EXPECT_EQ(1, stats.null_value_counts.at(2)); + if (stats.__isset.lower_bounds) { + EXPECT_EQ(0, stats.lower_bounds.count(2)); + } + if (stats.__isset.upper_bounds) { + EXPECT_EQ(0, stats.upper_bounds.count(2)); + } + + auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + const auto* root = reader->metadata()->schema()->group_node(); + ASSERT_EQ(1, root->field_count()); + const auto& payload = root->field(0); + ASSERT_NE(nullptr, payload->logical_type()); + EXPECT_TRUE(payload->logical_type()->is_variant()); + EXPECT_EQ(2, payload->field_id()); + const auto& payload_group = static_cast(*payload); + ASSERT_EQ(2, payload_group.field_count()); + EXPECT_EQ(-1, payload_group.field(0)->field_id()); + EXPECT_EQ(-1, payload_group.field(1)->field_id()); +} + +TEST_F(VParquetWriterTest, WritesNestedIcebergVariant) { + auto variant_type = std::make_shared(); + auto array_type = std::make_shared(variant_type); + auto nullable_array_type = make_nullable(array_type); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {nullable_array_type}); + + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + { + "id": 2, + "name": "events", + "required": false, + "type": { + "type": "list", + "element-id": 3, + "element": "variant", + "element-required": false + } + } + ] + })"; + std::unique_ptr schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + ParquetFileOptions options {.compression_type = TParquetCompressionType::UNCOMPRESSED, + .parquet_version = TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; + VIcebergParquetWriter transformer(&state, file_writer.get(), output_exprs, {"events"}, false, + options, &schema_json, *schema); + ASSERT_TRUE(transformer.open().ok()); + + JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, + .throw_on_invalid_json = true, + .check_duplicate_json_path = false}); + for (std::string_view json : + {std::string_view {R"({"kind":"nested"})"}, std::string_view {"null"}, + std::string_view {"{}"}, std::string_view {"7"}}) { + encoder.add_json({json.data(), json.size()}); + } + auto variant_values = ColumnVariantV2::create(); + variant_values->insert_encoded_batch(encoder.finish_batch()); + auto element_nulls = ColumnUInt8::create(); + element_nulls->get_data().assign({0, 0, 1, 0}); + auto nullable_values = + ColumnNullable::create(std::move(variant_values), std::move(element_nulls)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({3, 4}); + auto events = ColumnArray::create(std::move(nullable_values), std::move(offsets)); + auto outer_nulls = ColumnUInt8::create(); + outer_nulls->get_data().assign({0, 1}); + auto nullable_events = ColumnNullable::create(std::move(events), std::move(outer_nulls)); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(nullable_events), nullable_array_type, "events")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + TIcebergColumnStats stats; + ASSERT_TRUE(transformer.collect_file_statistics_after_close(&stats).ok()); + EXPECT_EQ(0, stats.value_counts.count(-1)); + EXPECT_EQ(0, stats.null_value_counts.count(-1)); + ASSERT_EQ(1, stats.value_counts.count(3)); + EXPECT_EQ(0, stats.lower_bounds.count(3)); + EXPECT_EQ(0, stats.upper_bounds.count(3)); + + auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + const auto* root = reader->metadata()->schema()->group_node(); + const auto& events_group = static_cast(*root->field(0)); + const auto& list_group = + static_cast(*events_group.field(0)); + const auto& element = list_group.field(0); + ASSERT_NE(nullptr, element->logical_type()); + EXPECT_TRUE(element->logical_type()->is_variant()); + EXPECT_EQ(3, element->field_id()); +} + TEST_F(VParquetWriterTest, IcebergDecimalBoundsUseMinimalSignedBytes) { for (const auto& [unscaled, expected] : std::vector> {{0, std::string("\x00", 1)}, diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 09e103626bab79..9b70c0ada25388 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -1893,6 +1893,108 @@ TEST(IcebergV2ReaderTest, PreparesIcebergNonFiniteInitialDefaults) { } } +TEST(IcebergV2ReaderTest, AnnotateVariantInitialDefaultMustBeNull) { + const auto variant_type = make_nullable(std::make_shared()); + + auto optional_variant = + external_schema_field("payload", 1, {}, std::nullopt, + external_primitive_type(TPrimitiveType::VARIANT), false, true); + TFileScanRangeParams valid_scan_params; + valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); + valid_scan_params.__set_current_schema_id(100); + valid_scan_params.__set_history_schema_info( + {external_schema(100, {std::move(optional_variant)})}); + + ColumnDefinition valid_column; + valid_column.name = "payload"; + valid_column.type = variant_type; + ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column) + .ok()); + ASSERT_TRUE(valid_column.is_optional.has_value()); + EXPECT_TRUE(*valid_column.is_optional); + EXPECT_FALSE(valid_column.initial_default_value.has_value()); + EXPECT_EQ(valid_column.default_expr, nullptr); + ASSERT_TRUE(valid_context.schema_column.has_value()); + EXPECT_TRUE(valid_context.schema_column->type->equals(*variant_type)); + + auto malformed_variant = + external_schema_field("payload", 1, {}, "{\"source\":\"malformed\"}", + external_primitive_type(TPrimitiveType::VARIANT), false, true); + TFileScanRangeParams malformed_scan_params; + malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); + malformed_scan_params.__set_current_schema_id(101); + malformed_scan_params.__set_history_schema_info( + {external_schema(101, {std::move(malformed_variant)})}); + + ColumnDefinition malformed_column; + malformed_column.name = "payload"; + malformed_column.type = variant_type; + ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params}; + const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context, + &malformed_column); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"), + std::string::npos) + << status; +} + +TEST(IcebergV2ReaderTest, StructInitialDefaultKeepsVariantChildNull) { + const auto variant_type = make_nullable(std::make_shared()); + const auto struct_type = make_nullable( + std::make_shared(DataTypes {variant_type}, Strings {"payload"})); + + auto optional_variant = + external_schema_field("payload", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::VARIANT), false, true); + auto valid_struct = + external_struct_schema_field("event", 1, {std::move(optional_variant)}, true, "{}"); + TFileScanRangeParams valid_scan_params; + valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); + valid_scan_params.__set_current_schema_id(100); + valid_scan_params.__set_history_schema_info({external_schema(100, {std::move(valid_struct)})}); + + ColumnDefinition valid_column; + valid_column.name = "event"; + valid_column.type = struct_type; + ProjectedColumnBuildContext valid_context {.scan_params = &valid_scan_params}; + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(), &valid_context, &valid_column) + .ok()); + ASSERT_NE(valid_column.default_expr, nullptr); + const auto* literal = dynamic_cast(valid_column.default_expr->root().get()); + ASSERT_NE(literal, nullptr); + EXPECT_TRUE(literal->get_data_type()->equals(*struct_type)); + Field value; + literal->get_column_ptr()->get(0, value); + const auto& struct_fields = value.get(); + ASSERT_EQ(struct_fields.size(), 1); + EXPECT_TRUE(struct_fields[0].is_null()); + + auto malformed_variant = + external_schema_field("payload", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::VARIANT), false, true); + auto malformed_struct = external_struct_schema_field( + "event", 1, {std::move(malformed_variant)}, true, "{\"2\":{\"source\":\"malformed\"}}"); + TFileScanRangeParams malformed_scan_params; + malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2); + malformed_scan_params.__set_current_schema_id(101); + malformed_scan_params.__set_history_schema_info( + {external_schema(101, {std::move(malformed_struct)})}); + + ColumnDefinition malformed_column; + malformed_column.name = "event"; + malformed_column.type = struct_type; + ProjectedColumnBuildContext malformed_context {.scan_params = &malformed_scan_params}; + const auto status = reader.annotate_projected_column(TFileScanSlotInfo(), &malformed_context, + &malformed_column); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("VARIANT initial-default for field 'payload' must be NULL"), + std::string::npos) + << status; +} + TEST(IcebergV2ReaderTest, AnnotateBuildsComplexInitialDefaults) { const auto required_int_type = std::make_shared(); const auto optional_string_type = make_nullable(std::make_shared()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java index cb2d420ed6d48d..cd3478f1dc8a22 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogOps.java @@ -495,6 +495,9 @@ public Optional loadNamespaceLocation(String dbName) { public void addColumn(String dbName, String tableName, IcebergColumnChange column, ConnectorColumnPosition position) { withTable(dbName, tableName, table -> { + if (column.getType().isVariantType()) { + IcebergWriterHelper.validateVariantColumn(table, column.getName()); + } IcebergNestedColumnEvolution.validateNoCaseInsensitiveSiblingCollision( table.schema().asStruct(), "", column.getName(), null, "add"); UpdateSchema updateSchema = table.updateSchema(); @@ -509,6 +512,11 @@ public void addColumn(String dbName, String tableName, IcebergColumnChange colum @Override public void addColumns(String dbName, String tableName, List columns) { withTable(dbName, tableName, table -> { + for (IcebergColumnChange column : columns) { + if (column.getType().isVariantType()) { + IcebergWriterHelper.validateVariantColumn(table, column.getName()); + } + } IcebergNestedColumnEvolution.validateNoCaseInsensitiveTopLevelCollisions(table.schema(), columns); UpdateSchema updateSchema = table.updateSchema(); for (IcebergColumnChange column : columns) { @@ -560,7 +568,17 @@ public void modifyColumn(String dbName, String tableName, IcebergColumnChange co // #65329 omit-preserves: an omitted COMMENT keeps the field's current doc rather than clearing it // (mirror of IcebergNestedColumnEvolution.modifyColumn / legacy IcebergMetadataOps). String targetComment = commentSpecified ? column.getComment() : current.doc(); - if (newType.isPrimitiveType()) { + if (current.type().isVariantType() != newType.isVariantType()) { + throw new DorisConnectorException("Cannot change column type involving Iceberg VARIANT: " + + column.getName() + ": " + current.type() + " -> " + newType); + } + if (newType.isVariantType()) { + // A VARIANT modify changes only metadata, so it does not check the file format: an existing + // v3 ORC table can hold a VARIANT column although Doris writes VARIANT values only to Parquet. + if (!Objects.equals(current.doc(), targetComment)) { + updateSchema.updateColumnDoc(currentName, targetComment); + } + } else if (newType.isPrimitiveType()) { // Reject a complex -> primitive change with a clean message before iceberg's updateColumn leaks // a raw type-diff error — parity with IcebergNestedColumnEvolution.modifyColumn / legacy // IcebergMetadataOps (symmetric with the non-complex -> complex guard below). diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java index ec7cbd5cdda834..baf9f5136106f2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java @@ -1043,6 +1043,11 @@ public void createTable(ConnectorSession session, ConnectorCreateTableRequest re // CREATE properties (ConnectorFactory.createConnector(catalogProperty.getProperties())). Map tableProperties = IcebergSchemaBuilder.buildTableProperties(request.getProperties(), properties); + if (request.getColumns().stream().anyMatch(column -> IcebergSchemaBuilder.isVariant(column.getType()))) { + IcebergWriterHelper.validateVariantWriteFormat( + IcebergSchemaBuilder.getEffectiveFormatVersion(tableProperties, properties), + IcebergSchemaBuilder.getEffectiveFileFormat(tableProperties, properties)); + } applyCreateTableComment(tableProperties, request.getComment()); try { executeAuthenticated(() -> { @@ -1442,7 +1447,7 @@ public void addNestedColumn(ConnectorSession session, ConnectorTableHandle handl if (!column.isNullable()) { throw new DorisConnectorException("New nested field '" + path.getFullPath() + "' must be nullable"); } - Type icebergType = IcebergSchemaBuilder.buildColumnType(column.getType()); + Type icebergType = IcebergSchemaBuilder.buildNestedFieldType(column.getType()); IcebergColumnChange change = new IcebergColumnChange(path.getLeafName(), icebergType, column.getComment(), null, column.isNullable()); try { @@ -1529,7 +1534,7 @@ public void modifyNestedColumn(ConnectorSession session, ConnectorTableHandle ha } Type icebergType; try { - icebergType = IcebergSchemaBuilder.buildColumnType(column.getType()); + icebergType = IcebergSchemaBuilder.buildNestedFieldType(column.getType()); } catch (DorisConnectorException buildError) { // Preserve the complete target identity so error parity cannot bind a same-named top-level field. throw upgradeNestedModifyError(iceHandle, path, column, buildError); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java index c5860163cf363a..c6026ef1f015c1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java @@ -25,6 +25,7 @@ import org.apache.doris.connector.spi.ddl.ConnectorSortField; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowLevelOperationMode; @@ -71,6 +72,9 @@ public final class IcebergSchemaBuilder { private static final List ICEBERG_TABLE_LOCATION_CHILD_DIRS = java.util.Arrays.asList("data", "metadata"); + // The Flink and Spark nickname of write.format.default, which wins over it (as IcebergWriterHelper reads them). + private static final String WRITE_FORMAT = "write-format"; + static final String NESTED_VARIANT_DDL_ERROR = "Iceberg VARIANT DDL currently supports only top-level columns"; private IcebergSchemaBuilder() { } @@ -91,7 +95,7 @@ public static Schema buildSchema(List columns) { List fields = new ArrayList<>(columns.size()); for (int i = 0; i < columns.size(); i++) { ConnectorColumn col = columns.get(i); - Type type = convert(col.getType(), ids); + Type type = convertColumn(col.getType(), ids); if (col.isNullable()) { fields.add(Types.NestedField.optional(i, col.getName(), type, col.getComment())); } else { @@ -113,6 +117,9 @@ public static Schema buildSchema(List columns) { private static Type convert(ConnectorType type, IdAllocator ids) { String name = type.getTypeName().toUpperCase(Locale.ROOT); switch (name) { + case "VARIANT": + case "VARIANT_COMPUTE_V2": + throw new DorisConnectorException(NESTED_VARIANT_DDL_ERROR); case "ARRAY": { // Element type/ids first, then the list's element id (post-order, matching legacy visitor). Type element = convert(type.getChildren().get(0), ids); @@ -150,6 +157,17 @@ private static Type convert(ConnectorType type, IdAllocator ids) { } } + /** Converts the type of a top-level column, the only place a VARIANT may be declared. */ + private static Type convertColumn(ConnectorType type, IdAllocator ids) { + return isVariant(type) ? Types.VariantType.get() : convert(type, ids); + } + + /** Whether {@code type} is a VARIANT, as declared by DDL or read from an Iceberg table. */ + static boolean isVariant(ConnectorType type) { + String name = type.getTypeName().toUpperCase(Locale.ROOT); + return "VARIANT".equals(name) || "VARIANT_COMPUTE_V2".equals(name); + } + /** * Builds the Iceberg {@link PartitionSpec} against {@code schema} from the neutral partition spec. * String-driven port of {@code IcebergUtils.solveIcebergPartitionSpec}: each field's transform name + @@ -320,6 +338,33 @@ public static int getEffectiveFormatVersion(Map requestPropertie } } + /** + * The effective iceberg data file format for a CREATE TABLE: catalog {@code table-default} properties, then + * the table request, then catalog {@code table-override} properties, each overriding the former; of the + * resulting {@code write-format} nickname and standard {@code write.format.default}, the nickname wins, as + * for an existing table ({@link IcebergWriterHelper#getFileFormat}). Without either, Parquet. + */ + public static FileFormat getEffectiveFileFormat(Map requestProperties, + Map catalogProperties) { + Map effective = new HashMap<>(); + copyFileFormat(effective, catalogProperties, CatalogProperties.TABLE_DEFAULT_PREFIX); + copyFileFormat(effective, requestProperties, ""); + copyFileFormat(effective, catalogProperties, CatalogProperties.TABLE_OVERRIDE_PREFIX); + String fileFormat = effective.containsKey(WRITE_FORMAT) ? effective.get(WRITE_FORMAT) + : effective.getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, FileFormat.PARQUET.name()); + return IcebergWriterHelper.toFileFormat(fileFormat); + } + + private static void copyFileFormat(Map effective, Map properties, + String prefix) { + for (String property : new String[] {WRITE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT}) { + String value = properties.get(prefix + property); + if (value != null) { + effective.put(property, value); + } + } + } + /** * Builds the iceberg {@link Type} for a SINGLE column added/modified on an EXISTING table, reusing the * same neutral-type conversion as {@link #buildSchema} (scalars via {@link IcebergTypeMapping}, plus @@ -333,6 +378,15 @@ public static int getEffectiveFormatVersion(Map requestPropertie * @throws DorisConnectorException if the column type cannot be represented in iceberg */ public static Type buildColumnType(ConnectorType type) { + return convertColumn(type, new IdAllocator(0)); + } + + /** + * Like {@link #buildColumnType}, for a field nested in a STRUCT, which cannot be a VARIANT. + * + * @throws DorisConnectorException if the field type cannot be represented in iceberg + */ + public static Type buildNestedFieldType(ConnectorType type) { return convert(type, new IdAllocator(0)); } @@ -408,6 +462,8 @@ public static Literal parseDefaultLiteral(String value, Type type) { } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Decimal string: " + value, e); } + case VARIANT: + throw new IllegalArgumentException("Iceberg VARIANT default values must be NULL"); default: throw new IllegalArgumentException("Cannot parse unknown type: " + type); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java index 1e8600115b7e47..0ea68a006c7afe 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java @@ -521,6 +521,9 @@ private static TColumnType buildPrimitiveColumnType( } static String serializeInitialDefault(Type type, Object value, boolean enableTimestampTz) { + if (type.typeId() == TypeID.VARIANT) { + throw new DorisConnectorException("Iceberg VARIANT initial-default must be NULL"); + } if (type.isNestedType()) { return SingleValueParser.toJson(type, value); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java index 5a06993830c678..90358841810c0d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java @@ -66,6 +66,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.LocationUtil; +import org.apache.iceberg.util.PropertyUtil; import java.util.ArrayList; import java.util.Arrays; @@ -115,6 +116,8 @@ public ConnectorRowChangeStyle getRowChangeStyle() { // Older backends reject the iceberg_transform external sink partition function. private static final int SUPPORT_TRANSFORM_ROUTING_EXEC_VERSION = 18; private static final String TRANSFORM_PARTITION_FUNCTION = "iceberg_transform"; + // Shredded VARIANT values are written as typed Parquet columns, which Doris does not produce. + private static final String PARQUET_SHRED_VARIANTS = "write.parquet.shred-variants"; // Legacy IcebergUtils compression-codec property keys (connector-local copies; iceberg SDK has no // constant for the doris/spark-sql forms). @@ -192,7 +195,6 @@ public IcebergWritePlanProvider(IcebergCatalogProperties catalogProps, @Override public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandle handle) { - validateWriteSchema(handle); IcebergTableHandle tableHandle = (IcebergTableHandle) handle.getTableHandle(); IcebergConnectorTransaction transaction = currentTransaction(session); @@ -202,6 +204,10 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl // transaction switches on at commit time (Append vs ReplacePartitions / OverwriteFiles). IcebergWriteSchemaContext schemaContext = handle.getWriteOperation() == WriteOperation.REWRITE ? null : resolveWriteSchema(session, tableHandle, handle.getBranchName()); + if (schemaContext != null) { + // REWRITE resolves its schema after beginWrite and checks it in buildRewriteSink. + validateVariantWrite(handle, schemaContext); + } validateNestedPartitionWriteCompatibility(handle, schemaContext); IcebergWriteContext writeContext = buildWriteContext(handle, schemaContext); transaction.beginWrite(session, tableHandle.getDbName(), tableHandle.getTableName(), writeContext); @@ -390,22 +396,28 @@ private static boolean hasMeaningfulTypeParameters(String typeName) { || "TIMESTAMPTZ".equals(typeName); } - static void validateWriteSchema(List columns, boolean writesDataFiles) { - if (!writesDataFiles) { + /** + * Checks a write of data files whose target holds a VARIANT, at any depth, against the table it writes: + * VARIANT needs format-version 3, and Doris writes it only unshredded to Parquet data files. + */ + static void validateVariantWrite(List columns, boolean writesDataFiles, int formatVersion, + FileFormat fileFormat, Map writerProperties) { + if (!writesDataFiles || columns.stream().noneMatch(column -> containsVariant(column.getType()))) { return; } - if (columns.stream().anyMatch(column -> containsVariant(column.getType()))) { - // Reject the whole data-file write: validating only selected columns would let an - // unchanged Variant target flow through a writer that cannot preserve its physical identity. - throw new DorisConnectorException( - "Iceberg VARIANT columns are read-only and cannot be written"); + IcebergWriterHelper.validateVariantWriteFormat(formatVersion, fileFormat); + if (PropertyUtil.propertyAsBoolean(writerProperties, PARQUET_SHRED_VARIANTS, false)) { + throw new DorisConnectorException("Doris currently supports only unshredded Iceberg VARIANT writes; set " + + PARQUET_SHRED_VARIANTS + "=false before writing"); } } - static void validateWriteSchema(ConnectorWriteHandle handle) { + private static void validateVariantWrite(ConnectorWriteHandle handle, IcebergWriteSchemaContext schemaContext) { // The explicit INSERT column list can omit an unchanged Variant column, but the data writer // still binds the complete target schema and therefore must validate that complete shape. - validateWriteSchema(handle.getBoundTargetColumns(), handle.isWritesDataFiles()); + validateVariantWrite(handle.getBoundTargetColumns(), handle.isWritesDataFiles(), + schemaContext.getFormatVersion(), schemaContext.getFileFormat(), + schemaContext.getWriterProperties()); } private static void validateNestedPartitionWriteCompatibility( @@ -891,8 +903,9 @@ private TIcebergTableSink buildRewriteSink(Table table, IcebergTableHandle table IcebergWriteSchemaContext rewriteSchemaContext = IcebergWriteSchemaContext.create( table, tableHandle.getTableName(), Optional.empty(), mappingVarbinaryEnabled(), mappingTimestampTzEnabled()); - // REWRITE resolves its schema after beginWrite, so its rolling-upgrade fence must run here - // against the same partition spec that the compaction sink will actually use. + // REWRITE resolves its schema after beginWrite, so its Variant check and rolling-upgrade fence must run + // here against the same schema and partition spec that the compaction sink will actually use. + validateVariantWrite(handle, rewriteSchemaContext); validateNestedPartitionWriteCompatibility(handle, rewriteSchemaContext); TIcebergTableSink tSink = buildSink(table, tableHandle, handle, rewriteSchemaContext); tSink.setWriteType(TIcebergWriteType.REWRITE); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java index 12c592c8fa1ec8..ad535befa34bb5 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java @@ -474,6 +474,8 @@ static String toDorisSql(Type type, Object value, enableMappingVarbinary, enableMappingTimestampTz)); } return "named_struct(" + String.join(", ", fields) + ")"; + case VARIANT: + throw new DorisConnectorException("Iceberg VARIANT write-default must be NULL"); default: throw new DorisConnectorException("Unsupported Iceberg write-default type: " + type); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriterHelper.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriterHelper.java index a10a95cb45b379..29c9ba68fae612 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriterHelper.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriterHelper.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.thrift.TFileContent; import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; @@ -89,10 +90,38 @@ final class IcebergWriterHelper { // Local copy (value 3), mirroring the connector's per-class private ICEBERG_ROW_LINEAGE_MIN_VERSION in // IcebergConnectorMetadata; the self-contained helper does not import fe-core IcebergUtils. private static final int ICEBERG_ROW_LINEAGE_MIN_VERSION = 3; + // Iceberg introduced the VARIANT type in format-version 3. + private static final int ICEBERG_VARIANT_MIN_VERSION = 3; private IcebergWriterHelper() { } + /** + * Checks the format of a table that is to hold VARIANT values Doris writes: VARIANT needs format-version 3, + * and Doris writes it only to Parquet data files. + */ + static void validateVariantWriteFormat(int formatVersion, FileFormat fileFormat) { + if (formatVersion < ICEBERG_VARIANT_MIN_VERSION) { + throw new DorisConnectorException( + "Iceberg VARIANT writes require table format-version 3, but found " + formatVersion); + } + if (fileFormat != FileFormat.PARQUET) { + throw new DorisConnectorException( + "Iceberg VARIANT writes require Parquet data files, but found " + fileFormat); + } + } + + /** Checks that {@code table} can take the new VARIANT column {@code columnName} that Doris will write. */ + static void validateVariantColumn(Table table, String columnName) { + if (getFormatVersion(table) < ICEBERG_VARIANT_MIN_VERSION) { + throw new DorisConnectorException( + "Iceberg VARIANT column " + columnName + " requires table format-version 3"); + } + if (getFileFormat(table) != FileFormat.PARQUET) { + throw new DorisConnectorException("Iceberg VARIANT column " + columnName + " requires Parquet data files"); + } + } + /** * Converts the BE data-file commit fragments into an iceberg {@link WriteResult} of {@link DataFile}s * (the INSERT / OVERWRITE / MERGE-data path). A partitioned table requires non-empty partition values per @@ -413,7 +442,7 @@ static FileFormat getFileFormat(Table table, TableIdentifier id, IcebergFormatCa return toFileFormat(resolveFileFormatName(table, table.properties(), id, cache)); } - private static FileFormat toFileFormat(String fileFormatName) { + static FileFormat toFileFormat(String fileFormatName) { if (fileFormatName.toLowerCase().contains(ORC_NAME)) { return FileFormat.ORC; } else if (fileFormatName.toLowerCase().contains(PARQUET_NAME)) { @@ -560,8 +589,9 @@ private static boolean shouldCollectColumnStats( .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) != MetricsModes.None.get()); } + // A Parquet VARIANT reports the value and null counts of its own field id. return TypeUtil.indexById(writerSchema.asStruct()).values().stream() - .filter(field -> field.type().isPrimitiveType()) + .filter(field -> field.type().isPrimitiveType() || field.type().isVariantType()) .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) != MetricsModes.None.get()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java index e23b0cfb309aae..9ae6be015b26a6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/CatalogBackedIcebergCatalogOpsColumnEvolutionTest.java @@ -38,7 +38,9 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -333,6 +335,80 @@ public void testModifyMissingColumnFailsLoud() { Assertions.assertTrue(ex.getMessage().contains("does not exist")); } + // ---------- VARIANT ---------- + + /** Creates {@code db1.} with an id and, when {@code withVariant}, a required VARIANT payload. */ + private void createTable(String name, String formatVersion, String fileFormat, boolean withVariant) { + Map props = new HashMap<>(IcebergSchemaBuilder.buildTableProperties(Collections.emptyMap())); + props.put("format-version", formatVersion); + props.put("write.format.default", fileFormat); + Schema schema = withVariant + ? new Schema(Types.NestedField.optional(1, "id", Types.LongType.get()), + Types.NestedField.required(2, "payload", Types.VariantType.get(), "old")) + : new Schema(Types.NestedField.optional(1, "id", Types.LongType.get())); + ops.createTable("db1", name, schema, PartitionSpec.unpartitioned(), null, props); + } + + @Test + public void testAddVariantColumnNeedsFormatThreeParquetTable() { + // Doris writes VARIANT values only to format-version 3 Parquet data files, so a VARIANT column is added + // only to such a table, alone or with other columns. + createTable("v2_parquet", "2", "parquet", false); + createTable("v3_orc", "3", "orc", false); + createTable("v3_parquet", "3", "parquet", false); + IcebergColumnChange payload = change("payload", Types.VariantType.get(), "c", true); + DorisConnectorException v2 = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.addColumn("db1", "v2_parquet", payload, null)); + Assertions.assertEquals("Iceberg VARIANT column payload requires table format-version 3", v2.getMessage()); + DorisConnectorException orc = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.addColumns("db1", "v3_orc", Arrays.asList( + change("a", Types.IntegerType.get(), "c", true), payload))); + Assertions.assertEquals("Iceberg VARIANT column payload requires Parquet data files", orc.getMessage()); + Assertions.assertNull(ops.loadTable("db1", "v3_orc").schema().findField("a")); + + ops.addColumn("db1", "v3_parquet", payload, null); + ops.addColumns("db1", "v3_parquet", Arrays.asList( + change("a", Types.IntegerType.get(), "c", true), + change("other_payload", Types.VariantType.get(), "c", true))); + Schema schema = ops.loadTable("db1", "v3_parquet").schema(); + Assertions.assertEquals(Types.VariantType.get(), schema.findField("payload").type()); + Assertions.assertEquals(Types.VariantType.get(), schema.findField("other_payload").type()); + } + + @Test + public void testModifyVariantColumnChangesOnlyItsMetadata() { + // A VARIANT modify only changes metadata, so it skips the file format check: a v3 ORC table can hold a + // VARIANT column that another engine wrote. The comment changes, the column becomes nullable and moves, + // and its type stays VARIANT. + createTable("orc_variant", "3", "orc", true); + ops.modifyColumn("db1", "orc_variant", change("payload", Types.VariantType.get(), "new doc", true), true, + ConnectorColumnPosition.FIRST); + Schema schema = ops.loadTable("db1", "orc_variant").schema(); + Types.NestedField payload = schema.findField("payload"); + Assertions.assertEquals(Types.VariantType.get(), payload.type()); + Assertions.assertEquals("new doc", payload.doc()); + Assertions.assertTrue(payload.isOptional()); + Assertions.assertEquals("payload", schema.columns().get(0).name()); + } + + @Test + public void testModifyColumnTypeInvolvingVariantFailsLoud() { + createTable("parquet_variant", "3", "parquet", true); + DorisConnectorException toString = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.modifyColumn("db1", "parquet_variant", + change("payload", Types.StringType.get(), null, true), true, null)); + Assertions.assertEquals("Cannot change column type involving Iceberg VARIANT: payload: variant -> string", + toString.getMessage()); + DorisConnectorException toVariant = Assertions.assertThrows(DorisConnectorException.class, + () -> ops.modifyColumn("db1", "parquet_variant", + change("id", Types.VariantType.get(), null, true), true, null)); + Assertions.assertEquals("Cannot change column type involving Iceberg VARIANT: id: long -> variant", + toVariant.getMessage()); + Schema schema = ops.loadTable("db1", "parquet_variant").schema(); + Assertions.assertEquals(Types.VariantType.get(), schema.findField("payload").type()); + Assertions.assertEquals(Types.LongType.get(), schema.findField("id").type()); + } + // ---------- reorderColumns ---------- @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java index 62c7eb3ea41b55..49e6252f1cad63 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java @@ -30,6 +30,7 @@ import org.apache.doris.connector.spi.ddl.TagChange; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -402,6 +403,52 @@ private static void assertCatalogLevelV3Rejects(String catalogFormatVersionKey) Assertions.assertEquals(0, ctx.authCount); } + @Test + public void testCreateTableWithVariantNeedsFormatThreeParquet() { + // A VARIANT column needs a format-version 3 table that writes Parquet data files, whether the table request + // or the catalog's table-default / table-override sets them; the check runs before the remote seam. + Map v3Orc = new HashMap<>(); + v3Orc.put(TableProperties.FORMAT_VERSION, "3"); + v3Orc.put(TableProperties.DEFAULT_FILE_FORMAT, "orc"); + assertVariantCreateRejected(Collections.emptyMap(), Collections.emptyMap(), + "Iceberg VARIANT writes require table format-version 3, but found 2"); + assertVariantCreateRejected(v3Orc, Collections.emptyMap(), + "Iceberg VARIANT writes require Parquet data files, but found ORC"); + assertVariantCreateRejected(Collections.singletonMap(TableProperties.FORMAT_VERSION, "3"), + Collections.singletonMap("table-override." + TableProperties.DEFAULT_FILE_FORMAT, "orc"), + "Iceberg VARIANT writes require Parquet data files, but found ORC"); + + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + Map catalogProps = props(IcebergCatalogProperties.TYPE_REST); + catalogProps.put("table-default.format-version", "3"); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(catalogProps), new RecordingConnectorContext()) + .createTable(null, variantTable(Collections.emptyMap())); + Assertions.assertEquals(Types.VariantType.get(), ops.lastCreateSchema.findField("payload").type()); + } + + private static ConnectorCreateTableRequest variantTable(Map properties) { + return ConnectorCreateTableRequest.builder() + .dbName("db1").tableName("t1") + .columns(Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("BIGINT"), "", true, null, false), + new ConnectorColumn("payload", ConnectorType.of("VARIANT"), "", true, null, false))) + .properties(properties) + .build(); + } + + private static void assertVariantCreateRejected(Map properties, + Map catalogProperties, String message) { + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + Map catalogProps = props(IcebergCatalogProperties.TYPE_REST); + catalogProps.putAll(catalogProperties); + IcebergConnectorMetadata md = new IcebergConnectorMetadata( + ops, IcebergCatalogProperties.of(catalogProps), new RecordingConnectorContext()); + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> md.createTable(null, variantTable(properties))); + Assertions.assertEquals(message, ex.getMessage()); + Assertions.assertTrue(ops.log.isEmpty(), "reject must run before the remote seam"); + } + // ---------- dropTable ---------- @Test diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilderTest.java index 4eac0ff0abe308..4c8a29b006af1d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilderTest.java @@ -24,6 +24,8 @@ import org.apache.doris.connector.spi.ddl.ConnectorPartitionSpec; import org.apache.doris.connector.spi.ddl.ConnectorSortField; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -383,4 +385,70 @@ public void testTableFormatVersionWinsOverCatalogDefault() { Map props = IcebergSchemaBuilder.buildTableProperties(tableProps, catalogProps); Assertions.assertEquals("1", props.get(TableProperties.FORMAT_VERSION)); } + + @Test + public void variantIsSupportedOnlyAsTopLevelColumn() { + // A top-level VARIANT, as declared by DDL or as the VARIANT_COMPUTE_V2 type a CTAS reads from an Iceberg + // table, becomes an Iceberg VARIANT; one nested in a STRUCT, an ARRAY or a MAP is rejected, as is a + // nested field declared as VARIANT. + for (String typeName : Arrays.asList("VARIANT", "VARIANT_COMPUTE_V2")) { + ConnectorType variant = ConnectorType.of(typeName); + Schema schema = IcebergSchemaBuilder.buildSchema(Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), null, false, null), + new ConnectorColumn("payload", variant, "doc", true, null))); + Assertions.assertEquals(Types.VariantType.get(), schema.findField("payload").type()); + Assertions.assertTrue(schema.findField("payload").isOptional()); + Assertions.assertEquals(Types.VariantType.get(), IcebergSchemaBuilder.buildColumnType(variant)); + for (ConnectorType nested : Arrays.asList( + ConnectorType.structOf(Collections.singletonList("v"), Collections.singletonList(variant)), + ConnectorType.arrayOf(variant), + ConnectorType.mapOf(ConnectorType.of("STRING"), variant))) { + assertNestedVariantRejected(() -> IcebergSchemaBuilder.buildSchema(Collections.singletonList( + new ConnectorColumn("payload", nested, null, true, null)))); + assertNestedVariantRejected(() -> IcebergSchemaBuilder.buildColumnType(nested)); + } + assertNestedVariantRejected(() -> IcebergSchemaBuilder.buildNestedFieldType(variant)); + } + } + + private static void assertNestedVariantRejected(Runnable build) { + DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, build::run); + Assertions.assertEquals("Iceberg VARIANT DDL currently supports only top-level columns", + exception.getMessage()); + } + + @Test + public void effectiveFileFormatFollowsCatalogAndTablePrecedence() { + // Catalog table-default, then the table request, then catalog table-override; the write-format nickname wins + // over write.format.default wherever it is set, as it does for an existing table. + Assertions.assertEquals(FileFormat.PARQUET, + IcebergSchemaBuilder.getEffectiveFileFormat(Collections.emptyMap(), Collections.emptyMap())); + Assertions.assertEquals(FileFormat.ORC, IcebergSchemaBuilder.getEffectiveFileFormat( + Collections.singletonMap(TableProperties.DEFAULT_FILE_FORMAT, "orc"), Collections.emptyMap())); + Assertions.assertEquals(FileFormat.ORC, IcebergSchemaBuilder.getEffectiveFileFormat( + Collections.emptyMap(), Collections.singletonMap( + CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.DEFAULT_FILE_FORMAT, "orc"))); + Assertions.assertEquals(FileFormat.PARQUET, IcebergSchemaBuilder.getEffectiveFileFormat( + Collections.singletonMap(TableProperties.DEFAULT_FILE_FORMAT, "orc"), Collections.singletonMap( + CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.DEFAULT_FILE_FORMAT, "parquet"))); + Assertions.assertEquals(FileFormat.ORC, IcebergSchemaBuilder.getEffectiveFileFormat( + Collections.singletonMap("write-format", "orc"), Collections.singletonMap( + CatalogProperties.TABLE_OVERRIDE_PREFIX + TableProperties.DEFAULT_FILE_FORMAT, "parquet"))); + Assertions.assertEquals(FileFormat.PARQUET, IcebergSchemaBuilder.getEffectiveFileFormat( + Collections.singletonMap(TableProperties.DEFAULT_FILE_FORMAT, "orc"), Collections.singletonMap( + CatalogProperties.TABLE_OVERRIDE_PREFIX + "write-format", "parquet"))); + Map catalogDefaults = new HashMap<>(); + catalogDefaults.put(CatalogProperties.TABLE_DEFAULT_PREFIX + "write-format", "orc"); + catalogDefaults.put(CatalogProperties.TABLE_DEFAULT_PREFIX + TableProperties.DEFAULT_FILE_FORMAT, "parquet"); + Assertions.assertEquals(FileFormat.ORC, + IcebergSchemaBuilder.getEffectiveFileFormat(Collections.emptyMap(), catalogDefaults)); + } + + @Test + public void variantDefaultValuesMustBeNull() { + Assertions.assertNull(IcebergSchemaBuilder.parseDefaultLiteral(null, Types.VariantType.get())); + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergSchemaBuilder.parseDefaultLiteral("{\"source\":\"ddl\"}", Types.VariantType.get())); + Assertions.assertEquals("Iceberg VARIANT default values must be NULL", exception.getMessage()); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java index 0e7e317367845e..323ac026b8ad22 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java @@ -791,4 +791,14 @@ public void applyFailsLoudOnCorruptProp() { Assertions.assertThrows(RuntimeException.class, () -> IcebergSchemaUtils.applySchemaEvolution(params, "!!!not-base64-thrift!!!")); } + + @Test + public void variantInitialDefaultMustBeNull() { + // A VARIANT added without an initial-default reads as NULL from older data files, and any other + // initial-default is rejected rather than handed to BE. + Assertions.assertNull(IcebergSchemaUtils.writeDefaultToDorisString(Types.VariantType.get(), null, false)); + DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, + () -> IcebergSchemaUtils.serializeInitialDefault(Types.VariantType.get(), "non-null-variant", false)); + Assertions.assertEquals("Iceberg VARIANT initial-default must be NULL", exception.getMessage()); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java index b265391bb179e8..8fc9b5107a25c9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java @@ -113,47 +113,91 @@ public class IcebergWritePlanProviderTest { Collections.singletonMap("iceberg.catalog.type", "hadoop"); @Test - public void rejectsVariantDataWritesButAllowsDeleteOnlyMerge() { - ConnectorColumn nestedVariant = new ConnectorColumn("payload", - ConnectorType.structOf(Collections.singletonList("nested"), - Collections.singletonList(ConnectorType.of("VARIANT"))), - null, true, null); - Assertions.assertThrows(DorisConnectorException.class, - () -> IcebergWritePlanProvider.validateWriteSchema( - Collections.singletonList(nestedVariant), true)); - Assertions.assertDoesNotThrow(() -> IcebergWritePlanProvider.validateWriteSchema( - Collections.singletonList(nestedVariant), false)); - - ConnectorColumn id = new ConnectorColumn( - "id", ConnectorType.of("INT"), null, true, null); - WriteHandle partialInsert = new WriteHandle(new IcebergTableHandle("db1", "t2")) - .columns(Collections.singletonList(id)) - .boundTargetColumns(Arrays.asList(id, nestedVariant)); - Assertions.assertThrows(DorisConnectorException.class, - () -> IcebergWritePlanProvider.validateWriteSchema(partialInsert)); + public void variantDataWritesNeedUnshreddedFormatThreeParquetTables() { + // A write of data files whose target holds a VARIANT, at the root or nested, as plain VARIANT or as the + // VARIANT_COMPUTE_V2 carrier the engine hands to the write path, needs format-version 3 and unshredded + // Parquet data files. A write without data files (DELETE, a delete-only MERGE) and a target without a + // VARIANT are not checked. MUTATION: dropping the VARIANT_COMPUTE_V2 arm of containsVariant -> red. + ConnectorType carrier = ConnectorType.of("VARIANT_COMPUTE_V2"); + ConnectorColumn id = new ConnectorColumn("id", ConnectorType.of("INT"), null, true, null); + List variants = Arrays.asList( + new ConnectorColumn("v", ConnectorType.of("VARIANT"), null, true, null), + new ConnectorColumn("v", carrier, null, true, null), + new ConnectorColumn("payload", ConnectorType.structOf(Collections.singletonList("nested"), + Collections.singletonList(carrier)), null, true, null)); + Map unshredded = Collections.emptyMap(); + for (ConnectorColumn variant : variants) { + List columns = Arrays.asList(id, variant); + Assertions.assertDoesNotThrow(() -> IcebergWritePlanProvider.validateVariantWrite( + columns, true, 3, FileFormat.PARQUET, unshredded), variant.getType().toString()); + assertVariantWriteRejected(columns, 2, FileFormat.PARQUET, unshredded, + "Iceberg VARIANT writes require table format-version 3, but found 2"); + assertVariantWriteRejected(columns, 3, FileFormat.ORC, unshredded, + "Iceberg VARIANT writes require Parquet data files, but found ORC"); + assertVariantWriteRejected(columns, 3, FileFormat.PARQUET, + Collections.singletonMap("write.parquet.shred-variants", "true"), + "Doris currently supports only unshredded Iceberg VARIANT writes"); + Assertions.assertDoesNotThrow(() -> IcebergWritePlanProvider.validateVariantWrite( + columns, false, 2, FileFormat.ORC, unshredded), variant.getType().toString()); + } + Assertions.assertDoesNotThrow(() -> IcebergWritePlanProvider.validateVariantWrite( + Collections.singletonList(id), true, 2, FileFormat.ORC, unshredded)); + } + + private static void assertVariantWriteRejected(List columns, int formatVersion, + FileFormat fileFormat, Map writerProperties, String message) { + DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, + () -> IcebergWritePlanProvider.validateVariantWrite( + columns, true, formatVersion, fileFormat, writerProperties)); + Assertions.assertTrue(exception.getMessage().contains(message), exception.getMessage()); } @Test - public void rejectsRootAndNestedComputeVariantCarrier() { - // The engine hands an Iceberg VARIANT column to the write path as the VARIANT_COMPUTE_V2 carrier - // (ConnectorColumnConverter), not as plain VARIANT, so the read-only gate must recognize the carrier - // both as a root column and nested in a complex type. MUTATION: dropping the VARIANT_COMPUTE_V2 arm - // of containsVariant -> red. - ConnectorType carrier = ConnectorType.of("VARIANT_COMPUTE_V2"); - ConnectorColumn rootVariant = new ConnectorColumn("v", carrier, null, true, null); - ConnectorColumn nestedVariant = new ConnectorColumn("payload", - ConnectorType.structOf(Collections.singletonList("nested"), Collections.singletonList(carrier)), - null, true, null); - for (ConnectorColumn column : Arrays.asList(rootVariant, nestedVariant)) { - DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, - () -> IcebergWritePlanProvider.validateWriteSchema(Collections.singletonList(column), true), - column.getName()); - Assertions.assertTrue(exception.getMessage().contains("VARIANT") - && exception.getMessage().contains("read-only"), exception.getMessage()); - // A delete-only MERGE writes no data file, so the same schema stays usable for it. - Assertions.assertDoesNotThrow(() -> IcebergWritePlanProvider.validateWriteSchema( - Collections.singletonList(column), false), column.getName()); - } + public void planWriteWritesVariantsOnlyToFormatThreeParquetTables() { + // An INSERT whose column list omits the VARIANT column still binds the whole target schema, so the table it + // writes is checked. A v3 Parquet table takes the write, with the VARIANT in the schema JSON BE parses and + // column metrics collected for it although it is the only column with metrics, while a v3 ORC table and a + // shredding Parquet table reject it. MUTATION: collecting metrics only for primitive fields -> red. + Schema schema = new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "payload", Types.VariantType.get())); + InMemoryCatalog catalog = freshCatalog(); + Map props = new HashMap<>(); + props.put("format-version", "3"); + props.put("write.format.default", "parquet"); + props.put("write.data.path", "oss://bucket/wh/db1/variants/data"); + props.put("write.metadata.metrics.default", "none"); + props.put("write.metadata.metrics.column.payload", "counts"); + Table parquet = catalog.createTable(TableIdentifier.of("db1", "variants"), schema, + PartitionSpec.unpartitioned(), props); + TIcebergTableSink sink = planSink(parquet, contextWithStorage(), variantInsert(parquet, "variants")); + Assertions.assertTrue(sink.getSchemaJson().contains("\"type\":\"variant\""), sink.getSchemaJson()); + Assertions.assertTrue(sink.isCollectColumnStats()); + + props.put("write.format.default", "orc"); + Table orc = catalog.createTable(TableIdentifier.of("db1", "orc_variants"), schema, + PartitionSpec.unpartitioned(), props); + DorisConnectorException orcError = Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(orc, contextWithStorage(), variantInsert(orc, "orc_variants"))); + Assertions.assertTrue(orcError.getMessage().contains("require Parquet data files, but found ORC"), + orcError.getMessage()); + + props.put("write.format.default", "parquet"); + props.put("write.parquet.shred-variants", "true"); + Table shredded = catalog.createTable(TableIdentifier.of("db1", "shredded_variants"), schema, + PartitionSpec.unpartitioned(), props); + DorisConnectorException shredError = Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(shredded, contextWithStorage(), variantInsert(shredded, "shredded_variants"))); + Assertions.assertTrue(shredError.getMessage().contains("unshredded"), shredError.getMessage()); + } + + /** An INSERT into {@code id} only, bound to the whole write schema of the table. */ + private static WriteHandle variantInsert(Table table, String tableName) { + RecordingConnectorContext context = contextWithStorage(); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", tableName); + List columns = providerFor(table, context).getWriteColumns( + sessionFor(table, context), tableHandle, Optional.empty()).orElseThrow(AssertionError::new); + return new WriteHandle(tableHandle).columns(columns.subList(0, 1)).boundTargetColumns(columns); } private static InMemoryCatalog freshCatalog() { @@ -381,6 +425,33 @@ public void zonedAndNestedBinaryWriteDefaultsIgnoreLegacyMappingFlags() { IcebergWriteSchemaContext.toDorisSql(structType, structDefault, false, false)); } + @Test + public void variantWriteDefaultsMustBeNull() { + // An optional VARIANT column without a write-default writes NULL and a required one has no default, while a + // VARIANT write-default other than NULL is rejected; a STRUCT default keeps its VARIANT child NULL. + Schema schema = new Schema( + Types.NestedField.optional(1, "optional_payload", Types.VariantType.get()), + Types.NestedField.required(2, "required_payload", Types.VariantType.get())); + Table table = freshCatalog().createTable(TableIdentifier.of("db1", "variant_defaults"), schema, + PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "3")); + IcebergWriteSchemaContext writeContext = IcebergWriteSchemaContext.create( + table, "db1.variant_defaults", Optional.empty(), false, false); + Assertions.assertEquals("NULL", writeContext.getColumns().get(0).getDefaultValueSql()); + Assertions.assertNull(writeContext.getColumns().get(1).getDefaultValueSql()); + + DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, + () -> IcebergWriteSchemaContext.toDorisSql(Types.VariantType.get(), "{}", false, false)); + Assertions.assertTrue(exception.getMessage().contains("Iceberg VARIANT write-default must be NULL"), + exception.getMessage()); + + Types.StructType structType = Types.StructType.of( + Types.NestedField.optional(3, "payload", Types.VariantType.get())); + GenericRecord structDefault = GenericRecord.create(structType); + structDefault.set(0, null); + Assertions.assertEquals("named_struct('payload', NULL)", + IcebergWriteSchemaContext.toDorisSql(structType, structDefault, false, false)); + } + @Test public void branchWritePinsBranchHeadSchema() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java new file mode 100644 index 00000000000000..8efe75af278ff5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.nereids.CTEContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.VariantType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** Shared source-plan validation for external Variant sinks. */ +public final class VariantWritePlanValidator { + private VariantWritePlanValidator() { + } + + /** + * Rejects an implicit Variant-to-non-Variant cast in the lineage of a Variant target column. + * + *

Common-type analysis for UNION, IF and CASE runs before sink binding. Without this + * check, an object or array Variant can become SQL NULL while being cast to a scalar, and the + * sink will only see that scalar/NULL and encode it back as Variant. Explicit casts remain an + * intentional user conversion and are not rejected. Only the expressions whose values can reach + * the target are inspected: the condition of an IF or of a CASE branch only chooses a value, so a + * Variant path compared with a scalar there loses nothing that is written.

+ */ + public static void validateNoLossyCoercion(List targetColumns, Plan sourcePlan) { + validateNoLossyCoercion(targetColumns, sourcePlan, Optional.empty()); + } + + /** Validates a sink while its enclosing CTE producers still live in the analyzer context. */ + public static void validateNoLossyCoercion( + List targetColumns, Plan sourcePlan, CTEContext cteContext) { + validateNoLossyCoercion(targetColumns, sourcePlan, Optional.of(cteContext)); + } + + private static void validateNoLossyCoercion( + List targetColumns, Plan sourcePlan, Optional cteContext) { + if (targetColumns.size() != sourcePlan.getOutput().size()) { + throw new AnalysisException("VARIANT write target and source columns are not aligned"); + } + List sourceOrdinals = new ArrayList<>(targetColumns.size()); + for (int i = 0; i < targetColumns.size(); i++) { + sourceOrdinals.add(i); + } + validateNoLossyCoercion(targetColumns, sourcePlan, sourceOrdinals, cteContext); + } + + /** + * Validates target columns that are embedded in a wider source output, such as row-level DML + * where the operation and row-id routing columns precede the table data columns. + */ + public static void validateNoLossyCoercion( + List targetColumns, Plan sourcePlan, List sourceOrdinals) { + validateNoLossyCoercion(targetColumns, sourcePlan, sourceOrdinals, Optional.empty()); + } + + /** Validates mapped source ordinals with analyzer-time CTE producer visibility. */ + public static void validateNoLossyCoercion( + List targetColumns, Plan sourcePlan, List sourceOrdinals, + CTEContext cteContext) { + validateNoLossyCoercion(targetColumns, sourcePlan, sourceOrdinals, Optional.of(cteContext)); + } + + private static void validateNoLossyCoercion( + List targetColumns, Plan sourcePlan, List sourceOrdinals, + Optional cteContext) { + if (targetColumns.size() != sourceOrdinals.size()) { + throw new AnalysisException("VARIANT write target and source columns are not aligned"); + } + + List variantTargetOrdinals = new ArrayList<>(); + for (int i = 0; i < targetColumns.size(); i++) { + DataType targetType = DataType.fromCatalogType(targetColumns.get(i).getType()); + if (VariantType.containsVariant(targetType)) { + variantTargetOrdinals.add(i); + } + } + if (variantTargetOrdinals.isEmpty()) { + return; + } + + TraceContext traceContext = new TraceContext(sourcePlan, cteContext); + List sourceOutputs = sourcePlan.getOutput(); + for (int targetOrdinal : variantTargetOrdinals) { + int sourceOrdinal = sourceOrdinals.get(targetOrdinal); + if (sourceOrdinal < 0 || sourceOrdinal >= sourceOutputs.size()) { + throw new AnalysisException("VARIANT write target and source columns are not aligned"); + } + Column targetColumn = targetColumns.get(targetOrdinal); + traceOutputLineage( + sourcePlan, + Collections.singleton(sourceOutputs.get(sourceOrdinal).getExprId()), + targetColumn.getName(), + traceContext); + } + } + + /** + * Validates the requested outputs and returns lineage ExprIds supplied by an outer scope. + * Scalar subqueries use the returned IDs to continue tracing correlated columns in the + * enclosing plan. + */ + private static Set traceOutputLineage( + Plan plan, Set requiredExprIds, String targetColumn, TraceContext context) { + if (requiredExprIds.isEmpty()) { + return Collections.emptySet(); + } + if (plan instanceof SetOperation) { + return traceSetOperation(plan, (SetOperation) plan, requiredExprIds, targetColumn, context); + } + if (plan instanceof RecursiveCte) { + return traceRecursiveCte(plan, (RecursiveCte) plan, requiredExprIds, targetColumn, context); + } + if (plan instanceof LogicalGenerate) { + return traceGenerate((LogicalGenerate) plan, requiredExprIds, targetColumn, context); + } + if (plan instanceof LogicalCTEConsumer) { + return traceCteConsumer((LogicalCTEConsumer) plan, requiredExprIds, targetColumn, context); + } + + Set unresolvedExprIds = new HashSet<>(requiredExprIds); + Set inputExprIds = new HashSet<>(); + for (Expression expression : plan.getExpressions()) { + if (!(expression instanceof NamedExpression)) { + continue; + } + NamedExpression namedExpression = (NamedExpression) expression; + if (!requiredExprIds.contains(namedExpression.getExprId())) { + continue; + } + inputExprIds.addAll(validateExpression(namedExpression, targetColumn, context)); + unresolvedExprIds.remove(namedExpression.getExprId()); + } + inputExprIds.addAll(unresolvedExprIds); + + Set externalExprIds = new HashSet<>(inputExprIds); + for (Plan child : plan.children()) { + Set childOutputExprIds = child.getOutputExprIdSet(); + Set childRequiredExprIds = new HashSet<>(inputExprIds); + childRequiredExprIds.retainAll(childOutputExprIds); + externalExprIds.removeAll(childOutputExprIds); + externalExprIds.addAll(traceOutputLineage(child, childRequiredExprIds, targetColumn, context)); + } + return externalExprIds; + } + + private static Set traceRecursiveCte( + Plan plan, RecursiveCte recursiveCte, Set requiredExprIds, + String targetColumn, TraceContext context) { + Set externalExprIds = new HashSet<>(); + Set requiredOrdinals = requiredOutputOrdinals(plan, requiredExprIds); + for (int childIndex = 0; childIndex < plan.arity(); childIndex++) { + List childOutputs = recursiveCte.getRegularChildOutput(childIndex); + Set childRequiredExprIds = new HashSet<>(); + for (int ordinal : requiredOrdinals) { + if (ordinal < childOutputs.size()) { + childRequiredExprIds.add(childOutputs.get(ordinal).getExprId()); + } + } + externalExprIds.addAll(traceOutputLineage( + plan.child(childIndex), childRequiredExprIds, targetColumn, context)); + } + return externalExprIds; + } + + private static Set traceGenerate( + LogicalGenerate generate, Set requiredExprIds, + String targetColumn, TraceContext context) { + Set childRequiredExprIds = new HashSet<>(requiredExprIds); + List generatorOutputs = generate.getGeneratorOutput(); + List generators = generate.getGenerators(); + for (int ordinal = 0; ordinal < generatorOutputs.size(); ordinal++) { + Slot generatorOutput = generatorOutputs.get(ordinal); + if (!requiredExprIds.contains(generatorOutput.getExprId())) { + continue; + } + childRequiredExprIds.remove(generatorOutput.getExprId()); + childRequiredExprIds.addAll(validateExpression(generators.get(ordinal), targetColumn, context)); + } + Set childOutputExprIds = generate.child().getOutputExprIdSet(); + Set externalExprIds = new HashSet<>(childRequiredExprIds); + externalExprIds.removeAll(childOutputExprIds); + childRequiredExprIds.retainAll(childOutputExprIds); + externalExprIds.addAll(traceOutputLineage( + generate.child(), childRequiredExprIds, targetColumn, context)); + return externalExprIds; + } + + private static Set traceSetOperation( + Plan plan, SetOperation setOperation, Set requiredExprIds, + String targetColumn, TraceContext context) { + Set externalExprIds = new HashSet<>(); + Set requiredOrdinals = requiredOutputOrdinals(plan, requiredExprIds); + + if (plan instanceof LogicalUnion) { + for (List constantRow : ((LogicalUnion) plan).getConstantExprsList()) { + for (int ordinal : requiredOrdinals) { + if (ordinal < constantRow.size()) { + externalExprIds.addAll(validateExpression(constantRow.get(ordinal), targetColumn, context)); + } + } + } + } + + for (int childIndex = 0; childIndex < setOperation.getArity(); childIndex++) { + List childOutputs = setOperation.getRegularChildOutput(childIndex); + Set childRequiredExprIds = new HashSet<>(); + for (int ordinal : requiredOrdinals) { + if (ordinal < childOutputs.size()) { + childRequiredExprIds.add(childOutputs.get(ordinal).getExprId()); + } + } + externalExprIds.addAll(traceOutputLineage( + plan.child(childIndex), childRequiredExprIds, targetColumn, context)); + } + return externalExprIds; + } + + private static Set requiredOutputOrdinals(Plan plan, Set requiredExprIds) { + Set requiredOrdinals = new HashSet<>(); + List outputs = plan.getOutput(); + for (int i = 0; i < outputs.size(); i++) { + if (requiredExprIds.contains(outputs.get(i).getExprId())) { + requiredOrdinals.add(i); + } + } + return requiredOrdinals; + } + + private static Set traceCteConsumer( + LogicalCTEConsumer consumer, Set requiredExprIds, + String targetColumn, TraceContext context) { + LogicalCTEProducer producer = context.cteProducers.get(consumer.getCteId()); + Optional producerPlan = producer == null + ? context.findAnalyzerCteProducer(consumer) + : Optional.of(producer.child()); + if (!producerPlan.isPresent() || !context.activeCteIds.add(consumer.getCteId())) { + return Collections.emptySet(); + } + try { + Set producerExprIds = new HashSet<>(); + for (Map.Entry mapping : consumer.getConsumerToProducerOutputMap().entrySet()) { + if (requiredExprIds.contains(mapping.getKey().getExprId())) { + producerExprIds.add(mapping.getValue().getExprId()); + } + } + return traceOutputLineage(producerPlan.get(), producerExprIds, targetColumn, context); + } finally { + context.activeCteIds.remove(consumer.getCteId()); + } + } + + /** + * Validates the casts on the values {@code expression} writes and returns the ExprIds those + * values come from: the slots they read, and the correlated slots of their scalar subqueries, + * which an outer scope may supply. + */ + private static Set validateExpression( + Expression expression, String targetColumn, TraceContext context) { + List valueNodes = valueNodes(expression); + validateLossyCast(valueNodes, targetColumn); + Set inputExprIds = new HashSet<>(); + for (Expression node : valueNodes) { + if (node instanceof Slot) { + inputExprIds.add(((Slot) node).getExprId()); + } else if (node instanceof ScalarSubquery) { + inputExprIds.addAll(traceScalarSubquery((ScalarSubquery) node, targetColumn, context)); + } + } + return inputExprIds; + } + + /** Traces a scalar subquery and returns its correlated slots that the subquery itself does not supply. */ + private static Set traceScalarSubquery( + ScalarSubquery scalarSubquery, String targetColumn, TraceContext context) { + scalarSubquery.getTypeCoercionExpr() + .ifPresent(coercion -> validateLossyCast(valueNodes(coercion), targetColumn)); + Plan queryPlan = scalarSubquery.getQueryPlan(); + context.collectCteProducers(queryPlan); + Set unresolvedExprIds = new HashSet<>(traceOutputLineage( + queryPlan, + Collections.singleton(queryPlan.getOutput().get(0).getExprId()), + targetColumn, + context)); + Set correlatedExprIds = new HashSet<>(); + for (Slot correlateSlot : scalarSubquery.getCorrelateSlots()) { + correlatedExprIds.add(correlateSlot.getExprId()); + } + unresolvedExprIds.retainAll(correlatedExprIds); + return unresolvedExprIds; + } + + /** + * The nodes of {@code expression} whose values can reach its result, in pre-order: every node + * except those under the condition of an IF or of a CASE branch, which only chooses a value. + */ + private static List valueNodes(Expression expression) { + List nodes = new ArrayList<>(); + collectValueNodes(expression, nodes); + return nodes; + } + + private static void collectValueNodes(Expression expression, List nodes) { + nodes.add(expression); + if (expression instanceof If) { + collectValueNodes(((If) expression).getTrueValue(), nodes); + collectValueNodes(((If) expression).getFalseValue(), nodes); + } else if (expression instanceof WhenClause) { + collectValueNodes(((WhenClause) expression).getResult(), nodes); + } else { + for (Expression child : expression.children()) { + collectValueNodes(child, nodes); + } + } + } + + private static void validateLossyCast(List valueNodes, String targetColumn) { + for (Expression node : valueNodes) { + if (!(node instanceof Cast)) { + continue; + } + Cast cast = (Cast) node; + if (!cast.isExplicitType() && losesVariantLeaf(cast.child().getDataType(), cast.getDataType())) { + throw new AnalysisException( + "VARIANT write cannot safely convert input column '" + targetColumn + + "': source expression implicitly casts VARIANT to " + + cast.getDataType().toSql() + " before sink analysis"); + } + } + } + + private static boolean losesVariantLeaf(DataType sourceType, DataType targetType) { + // Encoding a scalar or supported ARRAY value as one top-level Variant preserves all + // nested Variant leaves. The lossy case is a Variant leaf coerced to a non-Variant path. + if (targetType instanceof VariantType) { + return false; + } + if (sourceType instanceof VariantType) { + return true; + } + if (sourceType instanceof ArrayType) { + return targetType instanceof ArrayType + ? losesVariantLeaf( + ((ArrayType) sourceType).getItemType(), + ((ArrayType) targetType).getItemType()) + : VariantType.containsVariant(sourceType); + } + if (sourceType instanceof MapType) { + if (!(targetType instanceof MapType)) { + return VariantType.containsVariant(sourceType); + } + MapType sourceMap = (MapType) sourceType; + MapType targetMap = (MapType) targetType; + return losesVariantLeaf(sourceMap.getKeyType(), targetMap.getKeyType()) + || losesVariantLeaf(sourceMap.getValueType(), targetMap.getValueType()); + } + if (sourceType instanceof StructType) { + if (!(targetType instanceof StructType)) { + return VariantType.containsVariant(sourceType); + } + List sourceFields = ((StructType) sourceType).getFields(); + List targetFields = ((StructType) targetType).getFields(); + if (sourceFields.size() != targetFields.size()) { + return VariantType.containsVariant(sourceType); + } + for (int i = 0; i < sourceFields.size(); i++) { + if (losesVariantLeaf(sourceFields.get(i).getDataType(), targetFields.get(i).getDataType())) { + return true; + } + } + } + return false; + } + + private static final class TraceContext { + private final Map> cteProducers = new HashMap<>(); + private final Set activeCteIds = new HashSet<>(); + private final Optional analyzerCteContext; + + private TraceContext(Plan sourcePlan, Optional analyzerCteContext) { + this.analyzerCteContext = analyzerCteContext; + collectCteProducers(sourcePlan); + } + + private Optional findAnalyzerCteProducer(LogicalCTEConsumer consumer) { + return analyzerCteContext + .flatMap(context -> context.findCTEContext(consumer.getName())) + .filter(context -> context.getCteId().equals(consumer.getCteId())) + .flatMap(context -> context.getAnalyzedCTEPlan(consumer.getName())) + .map(Plan.class::cast); + } + + private void collectCteProducers(Plan plan) { + if (plan instanceof LogicalCTEProducer) { + LogicalCTEProducer producer = (LogicalCTEProducer) plan; + cteProducers.put(producer.getCteId(), producer); + } + for (Plan child : plan.children()) { + collectCteProducers(child); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 49833b0b5e1226..456b7ca1892e8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionRegistry; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.VariantWritePlanValidator; +import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.SqlCacheContext; import org.apache.doris.nereids.StatementContext; @@ -298,76 +300,80 @@ private LogicalExternalRowLevelMergeSink bindIcebergMergeSink( List outputExprs = sink.child().getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - List visibleColumns = sink.getCols().stream() - .filter(Column::isVisible) - .collect(ImmutableList.toImmutableList()); - // The connector-reserved passthrough columns (iceberg v3 row-lineage) are the hidden target columns - // marked reservedPassthrough. Derive their names from the sink's target schema so the meta-column - // check below stays name-based here (this site sees output-expression names, not Column objects) while - // fe-core no longer string-matches the iceberg column names. - List reservedPassthroughNames = sink.getCols().stream() - .filter(Column::isReservedPassthrough) - .map(Column::getName) - .collect(ImmutableList.toImmutableList()); - int dataExprCount = 0; - for (NamedExpression expr : outputExprs) { - if (!isIcebergMergeMetaColumn(expr.getName(), reservedPassthroughNames)) { - dataExprCount++; - } + if (sink.isWritesDataFiles()) { + validateMergeNoLossyCoercion(sink.getCols(), sink.child(), ctx.cascadesContext.getCteContext()); } - if (dataExprCount != visibleColumns.size()) { + List castExprs = coerceMergeOutput(sink.getCols(), outputExprs); + if (castExprs.equals(outputExprs)) { if (sink.getOutputExprs().equals(outputExprs)) { return sink; } return sink.withOutputExprs(outputExprs); } + LogicalProject project = new LogicalProject<>(castExprs, sink.child()); + return (LogicalExternalRowLevelMergeSink) sink.withChildAndUpdateOutput(project); + } - boolean changed = false; - int columnIndex = 0; - List castExprs = Lists.newArrayListWithCapacity(outputExprs.size()); - for (NamedExpression expr : outputExprs) { - if (isIcebergMergeMetaColumn(expr.getName(), reservedPassthroughNames)) { - castExprs.add(expr); + /** + * Rejects a lossy implicit cast in the lineage of a Variant target column of a merge sink that + * writes data files. See {@link #coerceMergeOutput} for the layout of the sink output. + */ + static void validateMergeNoLossyCoercion( + List sinkColumns, Plan sourcePlan, CTEContext cteContext) { + List targetColumns = Lists.newArrayList(); + List sourceOrdinals = Lists.newArrayList(); + int sourceOrdinal = 2; + for (Column column : sinkColumns) { + if (!column.isVisible() && !column.isReservedPassthrough()) { continue; } - Column column = visibleColumns.get(columnIndex++); - DataType targetType = DataType.fromCatalogType(column.getType()); - Expression castExpr = TypeCoercionUtils.castIfNotSameType(expr, targetType); - NamedExpression namedExpr; - if (castExpr instanceof NamedExpression) { - namedExpr = (NamedExpression) castExpr; - if (!column.getName().equalsIgnoreCase(namedExpr.getName())) { - namedExpr = new Alias(namedExpr, column.getName()); - } - } else { - namedExpr = new Alias(castExpr, column.getName()); - } - if (!namedExpr.equals(expr)) { - changed = true; - } - castExprs.add(namedExpr); - } - if (!changed) { - if (sink.getOutputExprs().equals(outputExprs)) { - return sink; + if (column.isVisible()) { + targetColumns.add(column); + sourceOrdinals.add(sourceOrdinal); } - return sink.withOutputExprs(outputExprs); + sourceOrdinal++; } - LogicalProject project = new LogicalProject<>(castExprs, sink.child()); - return (LogicalExternalRowLevelMergeSink) sink.withChildAndUpdateOutput(project); + VariantWritePlanValidator.validateNoLossyCoercion(targetColumns, sourcePlan, sourceOrdinals, cteContext); } - private boolean isIcebergMergeMetaColumn(String name, List reservedPassthroughNames) { - if (MergeOperation.OPERATION_COLUMN.equalsIgnoreCase(name)) { - return true; + /** + * Casts the visible data columns of a merge sink output to their target types. The output is the + * operation and row-id routing columns followed by one column per visible or connector-reserved + * passthrough target column, in target order. Columns are matched by position rather than by name, + * since a data column may be named like a routing column, such as `operation`. + */ + static List coerceMergeOutput(List sinkColumns, List outputExprs) { + List outputColumns = sinkColumns.stream() + .filter(column -> column.isVisible() || column.isReservedPassthrough()) + .collect(ImmutableList.toImmutableList()); + if (outputExprs.size() != 2 + outputColumns.size() + || !MergeOperation.OPERATION_COLUMN.equalsIgnoreCase(outputExprs.get(0).getName()) + || !Column.ICEBERG_ROWID_COL.equalsIgnoreCase(outputExprs.get(1).getName())) { + throw new AnalysisException("Row-level merge sink output is not aligned with its routing columns " + + "and target columns"); } - if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(name)) { - return true; + List castExprs = Lists.newArrayListWithCapacity(outputExprs.size()); + castExprs.add(outputExprs.get(0)); + castExprs.add(outputExprs.get(1)); + for (int i = 0; i < outputColumns.size(); ++i) { + Column column = outputColumns.get(i); + NamedExpression expr = outputExprs.get(i + 2); + castExprs.add(column.isVisible() ? coerceToColumn(expr, column) : expr); + } + return castExprs; + } + + private static NamedExpression coerceToColumn(NamedExpression expr, Column column) { + DataType targetType = DataType.fromCatalogType(column.getType()); + Expression castExpr = TypeCoercionUtils.castIfNotSameType(expr, targetType); + if (castExpr instanceof NamedExpression) { + NamedExpression namedExpr = (NamedExpression) castExpr; + if (column.getName().equalsIgnoreCase(namedExpr.getName())) { + return namedExpr; + } + return new Alias(namedExpr, column.getName()); } - // Connector-reserved passthrough columns (iceberg v3 row-lineage), matched case-insensitively by name - // — the names come from the sink's target Columns marked reservedPassthrough, so fe-core no longer - // knows the iceberg column names. The reserved set is tiny (0-2), so a linear scan is fine. - return reservedPassthroughNames.stream().anyMatch(n -> n.equalsIgnoreCase(name)); + return new Alias(castExpr, column.getName()); } private static boolean hasUnboundPlan(Plan plan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 5a235fd995bcee..9774f1c946749d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -40,6 +40,7 @@ import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.write.ConnectorChangelogMode; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.VariantWritePlanValidator; import org.apache.doris.datasource.connector.converter.ConnectorWriteValueConverter; import org.apache.doris.datasource.doris.RemoteDorisExternalTable; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; @@ -122,6 +123,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * bind an unbound logicalTableSink represent the target table of an insert command @@ -954,6 +956,11 @@ private Plan bindConnectorTableSink(MatchingContext tableColumnOrdinals = IntStream.rangeClosed(1, targetWriteSchema.size()).boxed() + .collect(ImmutableList.toImmutableList()); + VariantWritePlanValidator.validateNoLossyCoercion( + targetWriteSchema, child, tableColumnOrdinals, ctx.cascadesContext.getCteContext()); return new LogicalConnectorTableSink<>(database, table, targetWriteSchema, targetMetadata.getPartitionColumns(), targetMetadata.getWriteMetadataIdentity(), targetWriteSchema, outputExpressions, sink.getDMLCommandType(), false, @@ -991,6 +998,7 @@ private Plan bindConnectorTableSink(MatchingContext containsVariant(field.getDataType())); + } + return false; + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java new file mode 100644 index 00000000000000..973888bad03960 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java @@ -0,0 +1,426 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.nereids.CTEContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.ConnectorComputeVariantType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.function.Function; + +public class VariantWritePlanValidatorTest { + + @Test + public void testUnionIfAndCaseImplicitVariantLossAreRejected() { + List lossyPlans = ImmutableList.of( + lossyUnionPlan(), + projectExpression(variant -> new Cast(variant, IntegerType.INSTANCE)), + projectExpression(variant -> new If( + BooleanLiteral.TRUE, + new Cast(variant, IntegerType.INSTANCE), + new IntegerLiteral(1))), + projectExpression(variant -> new CaseWhen( + ImmutableList.of(new WhenClause( + BooleanLiteral.TRUE, + new Cast(variant, IntegerType.INSTANCE))), + new IntegerLiteral(1)))); + + for (Plan plan : lossyPlans) { + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), plan)); + Assertions.assertTrue(exception.getMessage().contains( + "VARIANT write cannot safely convert input column 'payload'"), exception.getMessage()); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + } + + @Test + public void testAnalyzedUnionIfAndCaseObjectAndArrayPlansAreRejected() { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + List sqlStatements = ImmutableList.of( + "SELECT 1, parse_to_variant('{\"kind\":\"union\"}') " + + "UNION ALL SELECT 2, 1", + "SELECT 3, parse_to_variant('[\"union\", 3]') " + + "UNION ALL SELECT 4, 1", + "SELECT 3, IF(TRUE, parse_to_variant('{\"kind\":\"if\"}'), 1)", + "SELECT 4, IF(TRUE, parse_to_variant('[\"if\", 4]'), 1)", + "SELECT 5, CASE WHEN TRUE THEN parse_to_variant('{\"kind\":\"case\"}') " + + "ELSE 1 END", + "SELECT 6, CASE WHEN TRUE THEN parse_to_variant('[\"case\", 6]') " + + "ELSE 1 END"); + + for (String sql : sqlStatements) { + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql).getPlan(); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan), + sql); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + } + } + + @Test + public void testAnalyzedExplicitVariantCastIsAccepted() { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + Plan analyzedPlan = PlanChecker.from(connectContext) + .analyze("SELECT 1, CAST(parse_to_variant('{\"explicit\":true}') AS INT)") + .getPlan(); + VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan); + } + + @Test + public void testAnalyzedCteProducerLossIsRejected() { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + Plan analyzedPlan = PlanChecker.from(connectContext).analyze( + "WITH source AS (SELECT CASE WHEN TRUE " + + "THEN parse_to_variant('{\"cte\":true}') ELSE 1 END AS payload) " + + "SELECT 1, payload FROM source").getPlan(); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan)); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + } + + @Test + public void testAnalyzedRecursiveUnionAndGenerateLossAreRejected() { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + List sqlStatements = ImmutableList.of( + "WITH RECURSIVE source AS (" + + "SELECT IF(TRUE, parse_to_variant(" + + "'{\"kind\":\"recursive\"}'), 1) AS payload " + + "UNION ALL SELECT CAST(1 AS DECIMAL(38, 9)) FROM source) " + + "SELECT 1, payload FROM source", + "SELECT 1, generated_payload FROM (SELECT 1 AS seed) source " + + "LATERAL VIEW explode(ARRAY(IF(TRUE, " + + "parse_to_variant('{\"kind\":\"generate\"}'), 1))) generated " + + "AS generated_payload"); + + for (String sql : sqlStatements) { + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql).getPlan(); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan), + sql); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + } + } + + @Test + public void testScalarSubqueryLossIsRejected() { + LogicalProject lossyQueryPlan = projectExpression(variant -> new If( + BooleanLiteral.TRUE, + new Cast(variant, IntegerType.INSTANCE), + new IntegerLiteral(1))); + LogicalOneRowRelation lossyQuerySource = oneRow( + new Alias(new ScalarSubquery(lossyQueryPlan), "payload")); + + ScalarSubquery coercedSubquery = (ScalarSubquery) new ScalarSubquery(variantInput()) + .withTypeCoercion(IntegerType.INSTANCE); + LogicalOneRowRelation coercedSubquerySource = oneRow( + new Alias(coercedSubquery, "payload")); + + for (Plan plan : ImmutableList.of(lossyQuerySource, coercedSubquerySource)) { + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), plan)); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + } + + @Test + public void testCorrelatedScalarSubqueryLossIsRejected() { + LogicalProject lossyOuterSource = projectExpression(variant -> new If( + BooleanLiteral.TRUE, + new Cast(variant, IntegerType.INSTANCE), + new IntegerLiteral(1))); + Slot correlatedSlot = lossyOuterSource.getOutput().get(0); + LogicalOneRowRelation correlatedQuery = oneRow( + new Alias(correlatedSlot, "correlated_payload")); + ScalarSubquery correlatedSubquery = new ScalarSubquery( + correlatedQuery, ImmutableList.of(correlatedSlot), false); + LogicalProject source = new LogicalProject<>( + ImmutableList.of(new Alias(correlatedSubquery, "payload")), + lossyOuterSource); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), source)); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + + @Test + public void testAnalyzerCteContextIsUsedBeforeAnchorConstruction() { + LogicalProject lossyProducerChild = projectExpression(variant -> new If( + BooleanLiteral.TRUE, + new Cast(variant, IntegerType.INSTANCE), + new IntegerLiteral(1))); + LogicalSubQueryAlias producerPlan = new LogicalSubQueryAlias<>( + "source", lossyProducerChild); + CTEId cteId = new CTEId(1); + CTEContext cteContext = new CTEContext(cteId, producerPlan, new CTEContext()); + cteContext.setAnalyzedPlan(producerPlan); + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + new RelationId(2), cteId, "source", producerPlan); + LogicalOneRowRelation source = oneRow( + new Alias(new ScalarSubquery(consumer), "payload")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), source, cteContext)); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + + @Test + public void testPrimitiveSourceAndExplicitVariantCastAreAccepted() { + LogicalOneRowRelation primitiveSource = oneRow( + new Alias(new IntegerLiteral(1), "payload")); + VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), primitiveSource); + + LogicalProject explicitCastSource = projectExpression( + variant -> new Cast(variant, IntegerType.INSTANCE, true)); + VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), explicitCastSource); + + LogicalOneRowRelation arrayVariantInput = oneRow(new Alias( + new NullLiteral(ArrayType.of(ConnectorComputeVariantType.INSTANCE)), "source_array")); + LogicalProject arrayToVariantSource = new LogicalProject<>(ImmutableList.of( + new Alias(new Cast( + arrayVariantInput.getOutput().get(0), + ConnectorComputeVariantType.INSTANCE), "payload")), + arrayVariantInput); + VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), arrayToVariantSource); + } + + @Test + public void testLossyCastOutsideVariantTargetLineageIsIgnored() { + LogicalOneRowRelation input = variantInput(); + Slot variantSlot = input.getOutput().get(0); + LogicalProject source = new LogicalProject<>(ImmutableList.of( + new Alias(new Cast(variantSlot, IntegerType.INSTANCE), "id"), + new Alias(variantSlot, "payload")), input); + List targets = ImmutableList.of( + new Column("id", org.apache.doris.catalog.Type.INT), + new Column("payload", ConnectorComputeVariantType.INSTANCE.toCatalogDataType())); + + VariantWritePlanValidator.validateNoLossyCoercion(targets, source); + } + + @Test + public void testIfAndCaseConditionsAreNotWrittenValues() { + // The condition of an IF or of a CASE branch only chooses the value written: a Variant path compared + // with a scalar there is implicitly cast, but nothing written is lost. MUTATION: searching the whole + // expression tree for lossy casts -> rejected. + List conditionCastPlans = ImmutableList.of( + projectExpression(variant -> new If( + new GreaterThan(new Cast(variant, IntegerType.INSTANCE), new IntegerLiteral(1)), + variant, + new NullLiteral(ConnectorComputeVariantType.INSTANCE))), + projectExpression(variant -> new CaseWhen( + ImmutableList.of(new WhenClause( + new GreaterThan(new Cast(variant, IntegerType.INSTANCE), new IntegerLiteral(1)), + variant)), + new NullLiteral(ConnectorComputeVariantType.INSTANCE)))); + for (Plan plan : conditionCastPlans) { + VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), plan); + } + + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + List sqlStatements = ImmutableList.of( + "SELECT 1, IF(payload['kind'] = 'event', payload, NULL) " + + "FROM (SELECT parse_to_variant('{\"kind\":\"event\"}') AS payload) source", + "SELECT 1, CASE WHEN payload['n'] > 1 THEN payload END " + + "FROM (SELECT parse_to_variant('{\"n\":2}') AS payload) source"); + for (String sql : sqlStatements) { + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql).getPlan(); + Assertions.assertDoesNotThrow( + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan), + sql); + } + } + + @Test + public void testColumnsReadOnlyByConditionsAreNotTraced() { + // A column the target value reads only through a condition is not traced further: the lossy cast that + // computes it is not written. MUTATION: tracing every input slot of the target expression -> rejected. + LogicalOneRowRelation input = variantInput(); + Slot variantSlot = input.getOutput().get(0); + LogicalProject matched = new LogicalProject<>(ImmutableList.of( + new Alias(variantSlot, "payload"), + new Alias(new GreaterThan(new Cast(variantSlot, IntegerType.INSTANCE), new IntegerLiteral(1)), + "matched")), input); + LogicalProject source = new LogicalProject<>(ImmutableList.of( + new Alias(new If( + matched.getOutput().get(1), + matched.getOutput().get(0), + new NullLiteral(ConnectorComputeVariantType.INSTANCE)), "payload")), matched); + VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), source); + + // A lossy cast the branch value reads through a column is still rejected. + LogicalProject lossyBranch = new LogicalProject<>(ImmutableList.of( + new Alias(new Cast(variantSlot, IntegerType.INSTANCE), "narrowed")), input); + LogicalProject lossySource = new LogicalProject<>(ImmutableList.of( + new Alias(new If( + BooleanLiteral.TRUE, + lossyBranch.getOutput().get(0), + new IntegerLiteral(1)), "payload")), lossyBranch); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), lossySource)); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + + @Test + public void testRowLevelDmlRoutingColumnsUseMappedSourceOrdinal() { + LogicalOneRowRelation input = variantInput(); + Slot variantSlot = input.getOutput().get(0); + LogicalProject source = new LogicalProject<>(ImmutableList.of( + new Alias(new IntegerLiteral(2), "operation"), + new Alias(new IntegerLiteral(7), "row_id"), + new Alias(new If( + BooleanLiteral.TRUE, + new Cast(variantSlot, IntegerType.INSTANCE), + new IntegerLiteral(1)), "payload")), input); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion( + variantTarget(), source, ImmutableList.of(2))); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + } + + @Test + public void testDefaultMappingRequiresSameOutputWidth() { + LogicalOneRowRelation source = oneRow( + new Alias(new IntegerLiteral(2), "operation"), + new Alias(new IntegerLiteral(7), "row_id"), + new Alias(new NullLiteral(ConnectorComputeVariantType.INSTANCE), "payload")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), source)); + Assertions.assertTrue(exception.getMessage().contains("target and source columns are not aligned"), + exception.getMessage()); + } + + @Test + public void testLossyCastInNestedVariantTargetIsRejected() { + ArrayType sourceType = ArrayType.of(ConnectorComputeVariantType.INSTANCE); + LogicalOneRowRelation input = oneRow(new Alias( + new NullLiteral(sourceType), "source_variants")); + LogicalProject source = new LogicalProject<>(ImmutableList.of( + new Alias(new Cast( + input.getOutput().get(0), ArrayType.of(IntegerType.INSTANCE)), "payloads")), + input); + List targets = ImmutableList.of(new Column( + "payloads", sourceType.toCatalogDataType())); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(targets, source)); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to ARRAY"), + exception.getMessage()); + } + + private static LogicalUnion lossyUnionPlan() { + LogicalProject left = projectExpression( + variant -> new Cast(variant, IntegerType.INSTANCE)); + LogicalOneRowRelation right = oneRow(new Alias(new IntegerLiteral(1), "payload")); + SlotReference output = new SlotReference("payload", IntegerType.INSTANCE); + return new LogicalUnion( + Qualifier.ALL, + ImmutableList.of(output), + ImmutableList.of( + ImmutableList.of((SlotReference) left.getOutput().get(0)), + ImmutableList.of((SlotReference) right.getOutput().get(0))), + ImmutableList.of(), + false, + ImmutableList.of(left, right)); + } + + private static LogicalProject projectExpression( + Function expressionFactory) { + LogicalOneRowRelation input = variantInput(); + return new LogicalProject<>( + ImmutableList.of(new Alias( + expressionFactory.apply(input.getOutput().get(0)), "payload")), + input); + } + + private static LogicalOneRowRelation variantInput() { + return oneRow(new Alias( + new NullLiteral(ConnectorComputeVariantType.INSTANCE), "source_variant")); + } + + private static LogicalOneRowRelation oneRow(NamedExpression... expressions) { + return new LogicalOneRowRelation( + new RelationId(1), ImmutableList.copyOf(expressions)); + } + + private static List variantTarget() { + return ImmutableList.of(new Column( + "payload", ConnectorComputeVariantType.INSTANCE.toCatalogDataType())); + } + + private static List variantTargetWithId() { + return ImmutableList.of( + new Column("id", org.apache.doris.catalog.Type.INT), + new Column("payload", ConnectorComputeVariantType.INSTANCE.toCatalogDataType())); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java index 56ab6bdb7a8ef9..5e76d20a3d43da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java @@ -17,24 +17,49 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.NereidsException; +import org.apache.doris.nereids.CTEContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.RulePromise; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.DayHourAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.HourSecondSub; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.expressions.functions.scalar.MicroSecondsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.MicroSecondsSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearMonthAdd; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.commands.merge.MergeOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.types.ConnectorComputeVariantType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; + class BindExpressionTest extends TestWithFeService implements GeneratedPlanPatterns { @Override @@ -248,4 +273,44 @@ public RulePromise defaultPromise() { return RulePromise.REWRITE; } + @Test + void testRowLevelMergeOutputIsCoercedByPosition() { + // A data column named like the operation routing column is matched by its position, so it is still cast to + // its target type. Matching by name skipped every cast of such an output. + List output = ImmutableList.of( + new Alias(new TinyIntLiteral((byte) 3), MergeOperation.OPERATION_COLUMN), + new SlotReference(Column.ICEBERG_ROWID_COL, StringType.INSTANCE), + new Alias(new IntegerLiteral(7), MergeOperation.OPERATION_COLUMN)); + List columns = ImmutableList.of( + new Column(MergeOperation.OPERATION_COLUMN, PrimitiveType.SMALLINT)); + + List coerced = BindExpression.coerceMergeOutput(columns, output); + + Assertions.assertEquals(3, coerced.size()); + Assertions.assertEquals(SmallIntType.INSTANCE, coerced.get(2).getDataType()); + Assertions.assertEquals(MergeOperation.OPERATION_COLUMN, coerced.get(2).getName()); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> BindExpression.coerceMergeOutput(columns, output.subList(0, 2))); + Assertions.assertTrue(exception.getMessage().contains("not aligned"), exception.getMessage()); + } + + @Test + void testRowLevelMergeValidatesVariantTargetNamedLikeTheOperationColumn() { + // The Variant target named like the operation routing column is traced at its own position. + LogicalOneRowRelation input = new LogicalOneRowRelation(new RelationId(1), ImmutableList.of( + new Alias(new NullLiteral(ConnectorComputeVariantType.INSTANCE), "source_payload"))); + Slot sourcePayload = input.getOutput().get(0); + LogicalProject source = new LogicalProject<>(ImmutableList.of( + new Alias(new TinyIntLiteral((byte) 3), MergeOperation.OPERATION_COLUMN), + new Alias(new StringLiteral("row"), Column.ICEBERG_ROWID_COL), + new Alias(new If(BooleanLiteral.TRUE, new Cast(sourcePayload, IntegerType.INSTANCE), + new IntegerLiteral(1)), MergeOperation.OPERATION_COLUMN)), input); + List columns = ImmutableList.of(new Column( + MergeOperation.OPERATION_COLUMN, ConnectorComputeVariantType.INSTANCE.toCatalogDataType())); + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> BindExpression.validateMergeNoLossyCoercion(columns, source, new CTEContext())); + Assertions.assertTrue(exception.getMessage().contains("input column 'operation'"), exception.getMessage()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java index cae860ff25352d..e6287d502c2154 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java @@ -28,7 +28,9 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ShortCircuitIf; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; @@ -38,6 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRowLevelMergeSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.ConnectorComputeVariantType; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.VarBinaryType; import org.apache.doris.qe.ConnectContext; @@ -140,4 +144,22 @@ public void mergeBranchesUseThePinnedWriterType() { Assertions.assertInstanceOf(ShortCircuitIf.class, projections.get(0).child(0), "Iceberg MERGE must evaluate only the selected branch regardless of the session setting"); } + + @Test + public void mergeBranchesAreCastToTheVariantTargetBeforeTheyAreSelected() { + // Each branch is cast to the Variant target before the branch selection, so selecting among a Variant and a + // numeric branch never derives a numeric common type that would turn an object or array Variant into NULL. + SlotReference targetPayload = new SlotReference("target_payload", ConnectorComputeVariantType.INSTANCE); + List projections = ExternalRowLevelMergePlanBuilder.generateFinalProjections( + ImmutableList.of("payload"), ImmutableList.of(ConnectorComputeVariantType.INSTANCE), + ImmutableList.of(ImmutableList.of(targetPayload), ImmutableList.of(new IntegerLiteral(1)))); + + Expression selection = projections.get(0).child(0); + Assertions.assertInstanceOf(ShortCircuitIf.class, selection); + Expression insertedBranch = selection.child(1); + Assertions.assertInstanceOf(Cast.class, insertedBranch); + Assertions.assertEquals(ConnectorComputeVariantType.INSTANCE, insertedBranch.getDataType()); + Assertions.assertEquals(IntegerType.INSTANCE, insertedBranch.child(0).getDataType()); + Assertions.assertEquals(ConnectorComputeVariantType.INSTANCE, selection.child(2).child(1).getDataType()); + } } diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out new file mode 100644 index 00000000000000..6e69c28cb0ce50 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_default_historical_row -- +1 true \N + +-- !variant_default_rows -- +1 true \N +2 true \N +3 true \N +4 true \N +5 true \N +6 true \N +7 false null +8 true \N + +-- !variant_default_counts -- +8 1 7 + +-- !variant_explicit_default_null_column -- +8 0 8 + +-- !variant_orc_historical_null -- +10 true \N + diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_read.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_read.out index ca9e630f18b677..b0c5b0bf9e8901 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_read.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_read.out @@ -127,6 +127,9 @@ true 5 170 4.17 -- !variant_nested_projection -- 1 first {"deep":{"name":"inside"},"x":11} {"kind":"open","score":101} {"enabled":true,"score":1001} 2 second \N null {"enabled":false,"score":2002} +3 doris-nested {"kind":"struct","n":3} null \N +4 empty {} \N \N +5 \N \N \N \N -- !variant_nested_filter -- 1 @@ -137,6 +140,9 @@ true 5 170 4.17 -- !variant_nested_expressions -- 1 11 inside open \N true 1002 2 \N \N \N 202 false 2003 +3 \N \N \N \N \N \N +4 \N \N \N \N \N \N +5 \N \N \N \N \N \N -- !variant_evolution_renamed_snapshot -- 1 initial 10 v1 @@ -185,3 +191,4 @@ true 5 170 4.17 -- !variant_mixed_format -- 1 {"format":"parquet"} 2 \N + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out index f1b82f09fbad1e..f2333c25116c9f 100644 --- a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out @@ -11,3 +11,9 @@ 28 false \N select-string 29 false \N false 30 false \N [3,4] + +-- !variant_dml_after_errors -- +1 {"kind":"updated"} +2 ["array",2] +3 {"kind":"object-source"} + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy new file mode 100644 index 00000000000000..93b367e7f6c702 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_variant_defaults", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String restUri = context.config.otherConfigs.get("iceberg_rest_uri") + if (restUri == null) { + restUri = "http://${externalEnvIp}:${restPort}" + } + String catalogName = "test_iceberg_variant_defaults" + String dbName = "iceberg_variant_defaults_db" + + spark_iceberg_multi """ + CREATE NAMESPACE IF NOT EXISTS demo.${dbName}; + DROP TABLE IF EXISTS demo.${dbName}.variant_defaults; + CREATE TABLE demo.${dbName}.variant_defaults (id INT) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.format.default'='parquet', + 'write.parquet.shred-variants'='false', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read' + ); + INSERT INTO demo.${dbName}.variant_defaults VALUES (1); + ALTER TABLE demo.${dbName}.variant_defaults ADD COLUMN payload VARIANT; + + DROP TABLE IF EXISTS demo.${dbName}.variant_defaults_orc_history; + CREATE TABLE demo.${dbName}.variant_defaults_orc_history (id INT) USING iceberg + TBLPROPERTIES ('format-version'='3', 'write.format.default'='orc'); + INSERT INTO demo.${dbName}.variant_defaults_orc_history VALUES (10); + ALTER TABLE demo.${dbName}.variant_defaults_orc_history ADD COLUMN payload VARIANT; + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='${restUri}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1', + 'meta.cache.iceberg.table.ttl-second'='0', + 'meta.cache.iceberg.schema.ttl-second'='0' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + sql """SET enable_file_scanner_v2=true""" + + order_qt_variant_default_historical_row """ + SELECT id, payload IS NULL, CAST(payload AS STRING) + FROM variant_defaults + ORDER BY id + """ + + sql """INSERT INTO variant_defaults (id) VALUES (2)""" + sql """INSERT INTO variant_defaults (id, payload) VALUES (3, DEFAULT)""" + sql """ + INSERT INTO variant_defaults VALUES + (4, PARSE_TO_VARIANT('{"source":"update"}')), + (5, PARSE_TO_VARIANT('{"source":"merge"}')) + """ + sql """UPDATE variant_defaults SET payload = DEFAULT(payload) WHERE id = 4""" + sql """ + MERGE INTO variant_defaults t + USING (SELECT 5 AS id UNION ALL SELECT 6 AS id) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET payload = DEFAULT(payload) + WHEN NOT MATCHED THEN INSERT (id, payload) VALUES (s.id, DEFAULT(payload)) + """ + sql """INSERT INTO variant_defaults VALUES (7, PARSE_TO_VARIANT('null'))""" + sql """ + INSERT INTO variant_defaults (id, payload) + SELECT 8, DEFAULT(payload) FROM variant_defaults WHERE id = 7 + """ + + order_qt_variant_default_rows """ + SELECT id, payload IS NULL, CAST(payload AS STRING) + FROM variant_defaults + ORDER BY id + """ + qt_variant_default_counts """ + SELECT COUNT(*), COUNT(payload), SUM(payload IS NULL) + FROM variant_defaults + """ + + sql """ALTER TABLE variant_defaults ADD COLUMN auxiliary VARIANT DEFAULT NULL""" + qt_variant_explicit_default_null_column """ + SELECT COUNT(*), COUNT(auxiliary), SUM(auxiliary IS NULL) + FROM variant_defaults + """ + + List> schemaBeforeInvalidDefault = sql """DESC variant_defaults""" + test { + sql """ + ALTER TABLE variant_defaults + ADD COLUMN invalid_payload VARIANT DEFAULT '{"invalid":true}' + """ + exception "support null" + } + assertEquals(schemaBeforeInvalidDefault, sql("DESC variant_defaults"), + "Rejected non-null VARIANT default changed the Iceberg schema") + + test { + sql """ + ALTER TABLE variant_defaults + ADD COLUMN required_payload VARIANT NOT NULL DEFAULT NULL + """ + exception "null default value" + } + assertEquals(schemaBeforeInvalidDefault, sql("DESC variant_defaults"), + "Rejected required VARIANT default changed the Iceberg schema") + + // Spark caches Iceberg snapshots, so refresh after Doris commits before cross-engine reads. + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_defaults""" + List> sparkCounts = spark_iceberg """ + SELECT COUNT(*), COUNT(payload), COUNT(auxiliary) + FROM demo.${dbName}.variant_defaults + """ + assertEquals(1, sparkCounts.size()) + assertEquals("8", sparkCounts[0][0].toString()) + assertEquals("1", sparkCounts[0][1].toString()) + assertEquals("0", sparkCounts[0][2].toString()) + + order_qt_variant_orc_historical_null """ + SELECT id, payload IS NULL, CAST(payload AS STRING) + FROM variant_defaults_orc_history + ORDER BY id + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy index 31d1f64925ee6d..461c530250d15d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_read.groovy @@ -211,6 +211,18 @@ suite("test_iceberg_variant_read", '{"n":', id, ',"padding":"', repeat('x', 256), '"}')) FROM range(0, 8192); + DROP TABLE IF EXISTS demo.${dbName}.variant_doris_multi_row_group; + CREATE TABLE demo.${dbName}.variant_doris_multi_row_group ( + id BIGINT, + info STRUCT, + padding STRING + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.format.default'='parquet', + 'write.parquet.shred-variants'='false' + ); + DROP TABLE IF EXISTS demo.${dbName}.variant_deletion_vector; CREATE TABLE demo.${dbName}.variant_deletion_vector (id INT, v VARIANT) USING iceberg TBLPROPERTIES ( @@ -267,6 +279,21 @@ suite("test_iceberg_variant_read", array(parse_json('null'), parse_json('{"kind":"close","score":202}')), map('primary', parse_json('{"enabled":false,"score":2002}')); + DROP TABLE IF EXISTS demo.${dbName}.variant_nested_deep; + CREATE TABLE demo.${dbName}.variant_nested_deep ( + id INT, + deep STRUCT< + level1: ARRAY< + MAP> + > + > + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.format.default'='parquet', + 'write.parquet.shred-variants'='false' + ); + DROP TABLE IF EXISTS demo.${dbName}.variant_signed_selector; CREATE TABLE demo.${dbName}.variant_signed_selector ( id INT, @@ -295,6 +322,33 @@ suite("test_iceberg_variant_read", CREATE TABLE demo.${dbName}.variant_write_guard (id INT) USING iceberg TBLPROPERTIES ('format-version'='3', 'write.format.default'='parquet'); INSERT INTO demo.${dbName}.variant_write_guard VALUES (1); + + DROP TABLE IF EXISTS demo.${dbName}.variant_coercion_guard; + CREATE TABLE demo.${dbName}.variant_coercion_guard ( + id INT, + payload VARIANT + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.format.default'='parquet', + 'write.parquet.shred-variants'='false' + ); + + DROP TABLE IF EXISTS demo.${dbName}.variant_operation_column; + CREATE TABLE demo.${dbName}.variant_operation_column ( + id INT, + `operation` VARIANT + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='3', + 'write.format.default'='parquet', + 'write.parquet.shred-variants'='false', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read' + ); + INSERT INTO demo.${dbName}.variant_operation_column + VALUES (1, parse_json('{"stage":"spark"}')); """ List> multiFileDataFiles = spark_iceberg """ @@ -448,6 +502,582 @@ public class AppendVariantEqualityDelete { sql """set enable_profile=true""" sql """set profile_level=2""" + // Doris currently writes only the unshredded metadata/value representation. Reject the + // table setting instead of silently ignoring the requested shredded encoding. + String shreddedWriteGuardSnapshot = latestSnapshotId("variant_page_pruning") + test { + sql """ + INSERT INTO variant_page_pruning VALUES + (6000, PARSE_TO_VARIANT('{"shredding":"unsupported"}')) + """ + exception "write.parquet.shred-variants=false" + } + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_page_pruning""" + assertEquals(shreddedWriteGuardSnapshot, latestSnapshotId("variant_page_pruning"), + "A rejected shredded Variant write must not commit a new snapshot") + + sql """DROP TABLE IF EXISTS variant_doris_write_v2""" + test { + sql """ + CREATE TABLE variant_doris_write_v2 ( + id INT, + payload VARIANT + ) ENGINE=ICEBERG + PROPERTIES ('format-version'='2', 'write.format.default'='parquet') + """ + exception "Iceberg VARIANT writes require table format-version 3" + } + sql """DROP TABLE IF EXISTS variant_doris_write_orc""" + test { + sql """ + CREATE TABLE variant_doris_write_orc ( + id INT, + payload VARIANT + ) ENGINE=ICEBERG + PROPERTIES ('format-version'='3', 'write.format.default'='orc') + """ + exception "Iceberg VARIANT writes require Parquet data files" + } + sql """DROP TABLE IF EXISTS variant_doris_nested_ddl_guard""" + test { + sql """ + CREATE TABLE variant_doris_nested_ddl_guard ( + id INT, + info STRUCT + ) ENGINE=ICEBERG + PROPERTIES ('format-version'='3', 'write.format.default'='parquet') + """ + exception "STRUCT unsupported sub-type: variant" + } + + sql """DROP TABLE IF EXISTS variant_doris_write""" + sql """ + CREATE TABLE variant_doris_write ( + id INT, + payload VARIANT + ) ENGINE=ICEBERG + PROPERTIES ('format-version'='3', 'write.format.default'='parquet') + """ + // Common-type analysis runs before sink binding. Reject implicit Variant-to-scalar casts + // instead of persisting an object/array as SQL NULL, while keeping ordinary scalar writes. + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT id, v FROM variant_values WHERE id = 1 + UNION ALL + SELECT 2, 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT id + 2, v FROM variant_root_arrays WHERE id = 1 + UNION ALL + SELECT 4, 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT 5, IF(TRUE, v, 1) FROM variant_values WHERE id = 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT 6, IF(TRUE, v, 1) FROM variant_root_arrays WHERE id = 2 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT 7, CASE WHEN TRUE THEN v ELSE 1 END + FROM variant_values WHERE id = 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT 8, CASE WHEN TRUE THEN v ELSE 1 END + FROM variant_root_arrays WHERE id = 2 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + WITH RECURSIVE source AS ( + SELECT IF(TRUE, v, 1) AS payload FROM variant_values WHERE id = 1 + UNION ALL + SELECT CAST(1 AS DECIMAL(38, 9)) FROM source + ) + SELECT 9, payload FROM source + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO variant_coercion_guard + SELECT 9, generated_payload + FROM (SELECT v FROM variant_values WHERE id = 1) source + LATERAL VIEW explode(ARRAY(IF(TRUE, v, 1))) generated AS generated_payload + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + + // Primitive-to-Variant coercion remains supported for both inline VALUES and ordinary SELECT. + sql """ + INSERT INTO variant_coercion_guard VALUES + (10, 7), + (11, 'values-string'), + (12, TRUE), + (13, ARRAY(1, 2)) + """ + sql """INSERT INTO variant_coercion_guard SELECT 20, 8""" + sql """INSERT INTO variant_coercion_guard SELECT 21, 'select-string'""" + sql """INSERT INTO variant_coercion_guard SELECT 22, FALSE""" + sql """INSERT INTO variant_coercion_guard SELECT 23, ARRAY(3, 4)""" + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_coercion_guard""" + List> sparkCoercionGuardRows = spark_iceberg """ + SELECT id, to_json(payload) + FROM demo.${dbName}.variant_coercion_guard + ORDER BY id + """ + assertEquals([ + ["10", "7"], + ["11", '"values-string"'], + ["12", "true"], + ["13", "[1,2]"], + ["20", "8"], + ["21", '"select-string"'], + ["22", "false"], + ["23", "[3,4]"] + ], sparkCoercionGuardRows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + + sql """ + INSERT INTO variant_doris_write VALUES + (1, PARSE_TO_VARIANT('{"name":"doris","n":20,"enabled":true,"ratio":1.25,"nested":{"city":"hangzhou"},"tags":["iceberg","spark"]}')), + (2, PARSE_TO_VARIANT('{"name":"second","n":-7,"enabled":false,"ratio":-3.5,"nested":{"city":"shanghai"},"tags":["doris","variant"]}')), + (3, PARSE_TO_VARIANT('{"name":"unicode-中文","n":0,"enabled":true,"ratio":0.0,"nested":{"city":null},"tags":[]}')), + (4, PARSE_TO_VARIANT('[1,"two",null,{"k":4}]')), + (5, PARSE_TO_VARIANT('42')), + (6, PARSE_TO_VARIANT('"root-string"')), + (7, PARSE_TO_VARIANT('true')), + (8, PARSE_TO_VARIANT('null')), + (9, NULL), + (10, PARSE_TO_VARIANT('"null"')), + (11, PARSE_TO_VARIANT('""')), + (12, PARSE_TO_VARIANT('{}')), + (13, PARSE_TO_VARIANT('[]')) + """ + + // A single Doris writer emits more than 128 MiB of logical data, forcing multiple Parquet + // row groups in one file. The nested Variant leaf verifies that Iceberg metrics are merged + // across every row group instead of taking statistics from only the first one. + sql """ + INSERT INTO variant_doris_multi_row_group + SELECT /*+ SET_VAR(parallel_pipeline_task_num=1) */ + number, + NAMED_STRUCT( + 'label', 'doris-row-group', + 'payload', IF( + number % 10 = 0, + CAST(NULL AS VARIANT), + PARSE_TO_VARIANT(CONCAT('{"kind":"doris","n":', number, '}')) + ) + ), + REPEAT('x', 2048) + FROM numbers("number" = "100000") + """ + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_doris_multi_row_group""" + // Iceberg allocates IDs to all top-level fields before nested fields; payload is 5, not 4. + int nestedPayloadFieldId = 5 + List> sparkDorisMultiRowGroupFiles = spark_iceberg """ + SELECT file_format, record_count, + value_counts[${nestedPayloadFieldId}], + null_value_counts[${nestedPayloadFieldId}], + column_sizes[${nestedPayloadFieldId}] IS NULL, + lower_bounds[${nestedPayloadFieldId}] IS NULL, + upper_bounds[${nestedPayloadFieldId}] IS NULL + FROM demo.${dbName}.variant_doris_multi_row_group.files + WHERE content = 0 + """ + assertEquals(1, sparkDorisMultiRowGroupFiles.size(), + "The Doris multi-row-group write must produce exactly one data file") + assertEquals([ + "parquet", "100000", "100000", "10000", "true", "true", "true" + ], sparkDorisMultiRowGroupFiles[0].collect { value -> + value == null ? null : value.toString().toLowerCase() + }) + List> sparkDorisMultiRowGroupValues = spark_iceberg """ + SELECT COUNT(*), COUNT(info.payload), + SUM(CASE WHEN info.payload IS NULL THEN 1 ELSE 0 END), + MIN(id), MAX(id), MIN(LENGTH(padding)), MAX(LENGTH(padding)) + FROM demo.${dbName}.variant_doris_multi_row_group + """ + assertEquals([ + "100000", "90000", "10000", "0", "99999", "2048", "2048" + ], sparkDorisMultiRowGroupValues[0].collect { value -> value.toString() }) + List> sparkDorisMultiRowGroupSamples = spark_iceberg """ + SELECT id, info.label, variant_get(info.payload, '\$.n', 'bigint') + FROM demo.${dbName}.variant_doris_multi_row_group + WHERE id IN (0, 1, 99999) + ORDER BY id + """ + assertEquals([ + ["0", "doris-row-group", null], + ["1", "doris-row-group", "1"], + ["99999", "doris-row-group", "99999"] + ], sparkDorisMultiRowGroupSamples.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + // CREATE/ADD followed by top-level MODIFY must remain metadata-only for Iceberg Variant. + sql """DROP TABLE IF EXISTS variant_doris_modify""" + sql """ + CREATE TABLE variant_doris_modify ( + id INT, + created_payload VARIANT NOT NULL + ) ENGINE=ICEBERG + PROPERTIES ('format-version'='3', 'write.format.default'='parquet') + """ + sql """ + ALTER TABLE variant_doris_modify + MODIFY COLUMN created_payload VARIANT NULL COMMENT 'created variant' + """ + sql """ALTER TABLE variant_doris_modify ADD COLUMN added_payload VARIANT""" + sql """ + ALTER TABLE variant_doris_modify + MODIFY COLUMN added_payload VARIANT COMMENT 'added variant' FIRST + """ + sql """ + INSERT INTO variant_doris_modify (id, created_payload, added_payload) + VALUES (1, PARSE_TO_VARIANT('{"source":"create"}'), + PARSE_TO_VARIANT('{"source":"add"}')) + """ + List> sparkModifiedVariantRows = spark_iceberg """ + SELECT id, + variant_get(created_payload, '\$.source', 'string'), + variant_get(added_payload, '\$.source', 'string') + FROM demo.${dbName}.variant_doris_modify + """ + assertEquals([["1", "create", "add"]], sparkModifiedVariantRows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + List> sparkDorisVariantRows = spark_iceberg """ + SELECT id, + variant_get(payload, '\$.name', 'string'), + variant_get(payload, '\$.n', 'int'), + variant_get(payload, '\$.enabled', 'boolean'), + variant_get(payload, '\$.ratio', 'double'), + variant_get(payload, '\$.nested.city', 'string'), + variant_get(payload, '\$.tags[1]', 'string'), + variant_get(payload, '\$[0]', 'int'), + variant_get(payload, '\$[1]', 'string'), + variant_get(payload, '\$[3].k', 'int'), + payload IS NULL + FROM demo.${dbName}.variant_doris_write + ORDER BY id + """ + List> sparkDorisVariantStrings = sparkDorisVariantRows.collect { row -> + row.collect { value -> value == null ? null : value.toString().toLowerCase() } + } + assertEquals([ + ["1", "doris", "20", "true", "1.25", "hangzhou", "spark", + null, null, null, "false"], + ["2", "second", "-7", "false", "-3.5", "shanghai", "variant", + null, null, null, "false"], + ["3", "unicode-中文", "0", "true", "0.0", null, null, + null, null, null, "false"], + ["4", null, null, null, null, null, null, "1", "two", "4", "false"], + ["5", null, null, null, null, null, null, null, null, null, "false"], + ["6", null, null, null, null, null, null, null, null, null, "false"], + ["7", null, null, null, null, null, null, null, null, null, "false"], + ["8", null, null, null, null, null, null, null, null, null, "false"], + ["9", null, null, null, null, null, null, null, null, null, "true"], + ["10", null, null, null, null, null, null, null, null, null, "false"], + ["11", null, null, null, null, null, null, null, null, null, "false"], + ["12", null, null, null, null, null, null, null, null, null, "false"], + ["13", null, null, null, null, null, null, null, null, null, "false"] + ], sparkDorisVariantStrings) + + // Keep SQL NULL, Variant null, the string "null", and empty values distinct. + List> sparkDorisRootRows = spark_iceberg """ + SELECT id, to_json(payload), payload IS NULL + FROM demo.${dbName}.variant_doris_write + WHERE id BETWEEN 5 AND 13 + ORDER BY id + """ + assertEquals([ + ["5", "42", "false"], + ["6", '"root-string"', "false"], + ["7", "true", "false"], + ["8", "null", "false"], + ["9", null, "true"], + ["10", '"null"', "false"], + ["11", '""', "false"], + ["12", "{}", "false"], + ["13", "[]", "false"] + ], sparkDorisRootRows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + + List> sparkDorisVariantFiles = spark_iceberg """ + SELECT file_format, record_count, + value_counts[2], null_value_counts[2], + lower_bounds[2] IS NULL, upper_bounds[2] IS NULL + FROM demo.${dbName}.variant_doris_write.files + WHERE content = 0 + """ + assertTrue(!sparkDorisVariantFiles.isEmpty()) + assertTrue(sparkDorisVariantFiles.every { row -> row[0].toString().equalsIgnoreCase("parquet") }) + assertEquals(13L, sparkDorisVariantFiles.sum { row -> Long.parseLong(row[1].toString()) }) + assertEquals(13L, sparkDorisVariantFiles.sum { row -> Long.parseLong(row[2].toString()) }) + assertEquals(1L, sparkDorisVariantFiles.sum { row -> Long.parseLong(row[3].toString()) }) + assertTrue(sparkDorisVariantFiles.every { row -> row[4].toString().equalsIgnoreCase("true") }) + assertTrue(sparkDorisVariantFiles.every { row -> row[5].toString().equalsIgnoreCase("true") }) + + // Doris writes Variant V2 leaves nested in every Iceberg complex container. Spark validates + // the committed Parquet Variant values, including SQL NULL versus Variant null and empties. + sql """ + INSERT INTO variant_nested VALUES + ( + 3, + NAMED_STRUCT( + 'label', 'doris-nested', + 'payload', PARSE_TO_VARIANT('{"kind":"struct","n":3}') + ), + ARRAY( + PARSE_TO_VARIANT('null'), + CAST(NULL AS VARIANT), + PARSE_TO_VARIANT('"null"'), + PARSE_TO_VARIANT('{}') + ), + MAP( + 'object', PARSE_TO_VARIANT('{"kind":"map","n":30}'), + 'json_null', PARSE_TO_VARIANT('null'), + 'sql_null', CAST(NULL AS VARIANT) + ) + ), + (4, NAMED_STRUCT('label', 'empty', 'payload', PARSE_TO_VARIANT('{}')), + ARRAY(), MAP()), + (5, NULL, NULL, NULL) + """ + // Spark caches Iceberg snapshots, so refresh after a Doris commit before cross-engine reads. + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_nested""" + + List> sparkNestedContainers = spark_iceberg """ + SELECT id, + info.label, + variant_get(info.payload, '\$.kind', 'string'), + variant_get(info.payload, '\$.n', 'int'), + info.payload IS NULL, + events IS NULL, size(events), + attrs IS NULL, size(attrs) + FROM demo.${dbName}.variant_nested + WHERE id IN (3, 4, 5) + ORDER BY id + """ + assertEquals([ + ["3", "doris-nested", "struct", "3", "false", "false", "4", "false", "3"], + ["4", "empty", null, null, "false", "false", "0", "false", "0"], + ["5", null, null, null, "true", "true", null, "true", null] + ], sparkNestedContainers.collect { row -> + row.collect { value -> value == null ? null : value.toString().toLowerCase() } + }) + + // Iceberg 1.10.1 cannot materialize Variant values from ArrayData/MapData in Spark 4. + // Doris still validates every nested value written to the shared Parquet files. + List> dorisNestedVariantValues = sql """ + SELECT id, + CAST(info.payload['kind'] AS STRING), + CAST(info.payload['n'] AS INT), + VARIANT_TYPE(events[1]), + events[2] IS NULL, + VARIANT_TYPE(events[3]), + CAST(events[3] AS STRING), + VARIANT_TYPE(events[4]), + CAST(attrs['object']['kind'] AS STRING), + CAST(attrs['object']['n'] AS INT), + VARIANT_TYPE(attrs['json_null']), + attrs['sql_null'] IS NULL + FROM variant_nested + WHERE id = 3 + """ + assertEquals([[ + "3", "struct", "3", "null", "true", "string", "null", "object", + "map", "30", "null", "true" + ]], dorisNestedVariantValues.collect { row -> + row.collect { value -> value == null ? null : value.toString().toLowerCase() } + }) + + sql """ + INSERT INTO variant_nested_deep VALUES + ( + 1, + NAMED_STRUCT( + 'level1', + ARRAY( + MAP( + 'outer', + NAMED_STRUCT( + 'note', 'from-doris', + 'payload', PARSE_TO_VARIANT( + '{"level2":{"level3":{"value":"deep-ok"}}}') + ) + ) + ) + ) + ), + ( + 2, + NAMED_STRUCT( + 'level1', + ARRAY(MAP( + 'outer', + NAMED_STRUCT('note', 'sql-null', 'payload', CAST(NULL AS VARIANT)) + )) + ) + ) + """ + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_nested_deep""" + List> sparkDeepContainerRows = spark_iceberg """ + SELECT id, + deep.level1[0]['outer'].note, + size(deep.level1) + FROM demo.${dbName}.variant_nested_deep + ORDER BY id + """ + assertEquals([ + ["1", "from-doris", "1"], + ["2", "sql-null", "1"] + ], sparkDeepContainerRows.collect { row -> + row.collect { value -> value == null ? null : value.toString().toLowerCase() } + }) + List> dorisDeepVariantRows = sql """ + SELECT id, + deep.level1[1]['outer'].note, + CAST(deep.level1[1]['outer'].payload['level2']['level3']['value'] AS STRING), + deep.level1[1]['outer'].payload IS NULL + FROM variant_nested_deep + ORDER BY id + """ + assertEquals([ + ["1", "from-doris", "deep-ok", "false"], + ["2", "sql-null", null, "true"] + ], dorisDeepVariantRows.collect { row -> + row.collect { value -> value == null ? null : value.toString().toLowerCase() } + }) + + // The first operation slot is merge-routing metadata; a quoted user column with the same + // name remains an ordinary data column and must still receive Variant V2 validation/coercion. + sql """ + UPDATE variant_operation_column + SET `operation` = PARSE_TO_VARIANT('{"stage":"doris-update","n":1}') + WHERE id = 1 + """ + sql """ + MERGE INTO variant_operation_column t + USING ( + SELECT 1 AS id, PARSE_TO_VARIANT('{"stage":"doris-merge","n":2}') AS payload + UNION ALL + SELECT 2 AS id, PARSE_TO_VARIANT('{"stage":"doris-insert","n":3}') AS payload + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET `operation` = s.payload + WHEN NOT MATCHED THEN INSERT (id, `operation`) VALUES (s.id, s.payload) + """ + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_operation_column""" + List> sparkOperationRows = spark_iceberg """ + SELECT id, + variant_get(`operation`, '\$.stage', 'string'), + variant_get(`operation`, '\$.n', 'int') + FROM demo.${dbName}.variant_operation_column + ORDER BY id + """ + assertEquals([ + ["1", "doris-merge", "2"], + ["2", "doris-insert", "3"] + ], sparkOperationRows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + + // Row-level DML must inspect the complete source lineage, not only the final sink-child type. + // Otherwise common-type coercion can turn an object Variant into SQL NULL before it is encoded. + test { + sql """ + UPDATE variant_operation_column + SET `operation` = IF(TRUE, `operation`, 1) + WHERE id = 1 + """ + exception "VARIANT write cannot safely convert input column 'operation'" + } + test { + sql """ + MERGE INTO variant_operation_column t + USING ( + SELECT 1 AS id, PARSE_TO_VARIANT('{"stage":"lossy-merge"}') AS payload + UNION ALL + SELECT 2 AS id, 1 AS payload + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET `operation` = s.payload + """ + exception "VARIANT write cannot safely convert input column 'operation'" + } + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_operation_column""" + List> sparkOperationRowsAfterRejectedDml = spark_iceberg """ + SELECT id, + variant_get(`operation`, '\$.stage', 'string'), + variant_get(`operation`, '\$.n', 'int') + FROM demo.${dbName}.variant_operation_column + ORDER BY id + """ + assertEquals([ + ["1", "doris-merge", "2"], + ["2", "doris-insert", "3"] + ], sparkOperationRowsAfterRejectedDml.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + + // Coerce each MERGE action to the Iceberg target before building the action IF. Otherwise + // the unchanged object Variant and primitive insert value may first be coerced to DECIMAL, + // turning the object into SQL NULL before the sink sees it. + sql """ + MERGE INTO variant_doris_write t + USING ( + SELECT 1 AS id + UNION ALL + SELECT 14 AS id + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET payload = t.payload + WHEN NOT MATCHED THEN INSERT (id, payload) VALUES (s.id, 1) + """ + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_doris_write""" + List> sparkUnchangedMergeVariant = spark_iceberg """ + SELECT variant_get(payload, '\$.name', 'string'), + variant_get(payload, '\$.n', 'int'), + variant_get(payload, '\$.nested.city', 'string') + FROM demo.${dbName}.variant_doris_write + WHERE id = 1 + """ + assertEquals([["doris", "20", "hangzhou"]], sparkUnchangedMergeVariant.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + List> sparkPrimitiveMergeVariant = spark_iceberg """ + SELECT to_json(payload) + FROM demo.${dbName}.variant_doris_write + WHERE id = 14 + """ + assertEquals("1", sparkPrimitiveMergeVariant[0][0].toString()) + def profileAction = new ProfileAction(context) def mergedProfile = { String profile -> if (!profile.contains("MergedProfile:")) { @@ -728,6 +1358,21 @@ public class AppendVariantEqualityDelete { FROM variant_type_matrix """ + String dorisMultiRowGroupToken = + "iceberg_variant_doris_multi_row_group_" + UUID.randomUUID().toString() + List> dorisMultiRowGroupRows = sql """ + SELECT '${dorisMultiRowGroupToken}', COUNT(*), SUM(LENGTH(padding)), + SUM(IF(info.payload IS NULL, 1, 0)) + FROM variant_doris_multi_row_group + """ + assertEquals([ + dorisMultiRowGroupToken, "100000", "204800000", "10000" + ], dorisMultiRowGroupRows[0].collect { value -> value.toString() }) + String dorisMultiRowGroupProfile = getProfileByToken( + dorisMultiRowGroupToken, ["RowGroupsTotalNum"]).toString() + assertTrue(counterSum(dorisMultiRowGroupProfile, "RowGroupsTotalNum") > 1, + "The single Doris-generated Parquet file did not contain multiple row groups") + String multiRowGroupColdToken = "iceberg_variant_multi_row_group_cold_" + UUID.randomUUID().toString() sql """ @@ -1008,17 +1653,21 @@ public class AppendVariantEqualityDelete { exception "payload" } - test { - sql """ - INSERT INTO variant_write_guard (id) - SELECT id - FROM variant_write_guard FOR VERSION AS OF ${writeGuardSourceSnapshot} - """ - exception "Iceberg VARIANT columns are read-only and cannot be written" - } + sql """ + INSERT INTO variant_write_guard (id) + SELECT id + FROM variant_write_guard FOR VERSION AS OF ${writeGuardSourceSnapshot} + """ + // Spark caches Iceberg snapshots, so refresh after Doris commits before cross-engine reads. + spark_iceberg """REFRESH TABLE demo.${dbName}.variant_write_guard""" + List> sparkPartialVariantRows = spark_iceberg """ + SELECT COUNT(*), COUNT(payload) + FROM demo.${dbName}.variant_write_guard + """ + assertEquals("2", sparkPartialVariantRows[0][0].toString()) + assertEquals("0", sparkPartialVariantRows[0][1].toString()) - // A delete-only MERGE emits only position deletes. It must remain available even though - // update/insert actions would route the unchanged Variant through the unsupported data writer. + // A delete-only MERGE still emits only position deletes and does not create a data file. String beforePositionDeleteSnapshot = latestSnapshotId("variant_values") sql """ MERGE INTO variant_values t @@ -1053,9 +1702,16 @@ public class AppendVariantEqualityDelete { // Files written before the Variant field existed have no physical Variant payload. Schema // evolution must synthesize NULL instead of rejecting their non-Parquet file format. + // Metadata-only MODIFY is valid on an existing Iceberg v3 ORC Variant table. Doris still + // rejects data writes because Iceberg ORC has no Variant writer. + sql """ALTER TABLE variant_orc MODIFY COLUMN v VARIANT COMMENT 'orc payload'""" order_qt_variant_orc_missing_column """ SELECT id, CAST(v AS STRING) FROM variant_orc ORDER BY id """ + test { + sql """INSERT INTO variant_orc VALUES (2, CAST('{"format":"orc"}' AS VARIANT))""" + exception "Iceberg VARIANT writes require Parquet data files" + } qt_variant_orc_count_star "SELECT COUNT(*) FROM variant_orc" order_qt_variant_mixed_format """ SELECT id, CAST(v AS STRING) FROM variant_mixed_format ORDER BY id diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy index 0ee47c376440c9..cde2e529c5e50a 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy @@ -50,6 +50,17 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { payload VARIANT ) USING paimon TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_dml_error; + CREATE TABLE paimon.${dbName}.t_variant_dml_error ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ( + 'file.format' = 'parquet', + 'primary-key' = 'id', + 'bucket' = '1' + ); """ sql """DROP CATALOG IF EXISTS ${catalogName}""" @@ -76,58 +87,81 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { (2, parse_to_variant('["array-source",2]')) """ - // Keep master's common-type semantics: mixed Variant and numeric expressions resolve to DECIMAL. - // Object and array Variant values therefore become SQL NULL before the connector sink converts - // the scalar result back to Paimon's Variant type. - sql """ - INSERT INTO t_variant_error - SELECT 100, payload FROM t_variant_coercion_source WHERE id = 1 - UNION ALL - SELECT 101, 1 - """ - sql """ - INSERT INTO t_variant_error - SELECT 102, payload FROM t_variant_coercion_source WHERE id = 2 - UNION ALL - SELECT 103, 1 - """ - sql """ - INSERT INTO t_variant_error - SELECT 104, IF(TRUE, payload, 1) - FROM t_variant_coercion_source WHERE id = 1 - """ - sql """ - INSERT INTO t_variant_error - SELECT 105, IF(TRUE, payload, 1) - FROM t_variant_coercion_source WHERE id = 2 - """ - sql """ - INSERT INTO t_variant_error - SELECT 106, CASE WHEN TRUE THEN payload ELSE 1 END - FROM t_variant_coercion_source WHERE id = 1 - """ - sql """ - INSERT INTO t_variant_error - SELECT 107, CASE WHEN TRUE THEN payload ELSE 1 END - FROM t_variant_coercion_source WHERE id = 2 - """ - sql """ - INSERT INTO t_variant_error - WITH RECURSIVE source(iter, payload) AS ( - SELECT CAST(0 AS INT), IF(TRUE, payload, 1) - FROM t_variant_coercion_source WHERE id = 1 + // The common type of a Variant and a number is DECIMAL. Writing it would turn an object or array Variant + // into SQL NULL before the sink converts the scalar back to Variant, so such writes are rejected. + test { + sql """ + INSERT INTO t_variant_error + SELECT 100, payload FROM t_variant_coercion_source WHERE id = 1 UNION ALL - SELECT CAST(iter + 1 AS INT), CAST(1 AS DECIMAL(38, 9)) - FROM source WHERE iter < 1 - ) - SELECT 108 + iter, payload FROM source - """ - sql """ - INSERT INTO t_variant_error - SELECT 110, generated_payload - FROM (SELECT payload FROM t_variant_coercion_source WHERE id = 1) source - LATERAL VIEW explode(ARRAY(IF(TRUE, payload, 1))) generated AS generated_payload - """ + SELECT 101, 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 102, payload FROM t_variant_coercion_source WHERE id = 2 + UNION ALL + SELECT 103, 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 104, IF(TRUE, payload, 1) + FROM t_variant_coercion_source WHERE id = 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 105, IF(TRUE, payload, 1) + FROM t_variant_coercion_source WHERE id = 2 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 106, CASE WHEN TRUE THEN payload ELSE 1 END + FROM t_variant_coercion_source WHERE id = 1 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 107, CASE WHEN TRUE THEN payload ELSE 1 END + FROM t_variant_coercion_source WHERE id = 2 + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + WITH RECURSIVE source(iter, payload) AS ( + SELECT CAST(0 AS INT), IF(TRUE, payload, 1) + FROM t_variant_coercion_source WHERE id = 1 + UNION ALL + SELECT CAST(iter + 1 AS INT), CAST(1 AS DECIMAL(38, 9)) + FROM source WHERE iter < 1 + ) + SELECT 108 + iter, payload FROM source + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 110, generated_payload + FROM (SELECT payload FROM t_variant_coercion_source WHERE id = 1) source + LATERAL VIEW explode(ARRAY(IF(TRUE, payload, 1))) generated AS generated_payload + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } sql """ INSERT INTO t_variant_error VALUES @@ -168,28 +202,6 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { row.collect { value -> value == null ? null : value.toString() } }) - def sparkCoercionRows = spark_paimon """ - SELECT id, to_json(payload) - FROM paimon.${dbName}.t_variant_error - WHERE id BETWEEN 100 AND 110 - ORDER BY id - """ - assertEquals([ - ["100", null], - ["101", "1"], - ["102", null], - ["103", "1"], - ["104", null], - ["105", null], - ["106", null], - ["107", null], - ["108", null], - ["109", "1"], - ["110", null] - ], sparkCoercionRows.collect { row -> - row.collect { value -> value == null ? null : value.toString() } - }) - // Invalid JSON is preserved as a Variant string unless the global // throw-on-invalid-JSON option is enabled. order_qt_variant_after_errors """ @@ -197,9 +209,46 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { CAST(payload['recovered'] AS BOOLEAN), payload FROM t_variant_error - WHERE id BETWEEN 20 AND 30 ORDER BY id """ + + // UPDATE and MERGE write their rows through the same check. A Variant compared with a scalar in the + // condition of an IF or a CASE only chooses the value written, so it is no lossy write. + sql """ + INSERT INTO t_variant_dml_error VALUES + (1, parse_to_variant('{"kind":"object"}')), + (2, parse_to_variant('["array",2]')) + """ + test { + sql """UPDATE t_variant_dml_error SET payload = IF(TRUE, payload, 1) WHERE id = 1""" + exception "VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + MERGE INTO t_variant_dml_error t + USING ( + SELECT 1 AS id, parse_to_variant('{"kind":"lossy-merge"}') AS payload + UNION ALL + SELECT 2 AS id, 1 AS payload + ) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET payload = s.payload + """ + exception "VARIANT write cannot safely convert input column 'payload'" + } + sql """ + UPDATE t_variant_dml_error + SET payload = IF(payload['kind'] = 'object', parse_to_variant('{"kind":"updated"}'), payload) + WHERE id = 1 + """ + sql """ + INSERT INTO t_variant_dml_error + SELECT 3, CASE WHEN payload['kind'] = 'object-source' THEN payload END + FROM t_variant_coercion_source WHERE id = 1 + """ + order_qt_variant_dml_after_errors """ + SELECT id, CAST(payload AS STRING) FROM t_variant_dml_error ORDER BY id + """ } finally { sql """SET force_jni_scanner = false""" sql """DROP CATALOG IF EXISTS ${catalogName}""" From 4a6b39396e5fbebdbfa05cc195cb3842e68f44e0 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Oct 2026 11:50:36 +0100 Subject: [PATCH 2/4] [fix](fe) Follow every path a Variant write takes through CTEs and set operations Four gaps in the Variant write validation: - The recursive term of a recursive CTE reads the previous iteration through the work table, so a column it writes can carry the value of another column. Each term was traced only for the requested column and the work-table columns it read were dropped, so a loss in another anchor column reached the target unchecked. Work-table columns are now mapped back to the CTE's columns and traced until no new one is reached. - A leading WITH binds its sink before the outer anchors exist, so its producer is taken from the analyzer context. The producers of the WITH clauses nested in it were never collected, their consumers resolved to nothing, and a loss inside them was accepted. They are now collected before the producer is traced. - A producer shared by several consumers was traced once for every path through the plan: a chain of CTEs that each read the previous one twice walked the first producer 2^N times. A completed trace is now reused for the same requested producer outputs. - EXCEPT and INTERSECT return rows of their first input; the later inputs only decide which rows stay. A lossy cast there writes nothing but rejected the statement. Only the first input is traced now, as the conditions of IF and CASE are not. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../datasource/VariantWritePlanValidator.java | 106 ++++++++++++--- .../VariantWritePlanValidatorTest.java | 122 ++++++++++++++++++ 2 files changed, 207 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java index 8efe75af278ff5..71dd5a4aba459d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.Column; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.CTEId; @@ -35,8 +36,12 @@ import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalWorkTableReference; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; @@ -44,6 +49,8 @@ import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.VariantType; +import com.google.common.collect.ImmutableSet; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -66,7 +73,8 @@ private VariantWritePlanValidator() { * sink will only see that scalar/NULL and encode it back as Variant. Explicit casts remain an * intentional user conversion and are not rejected. Only the expressions whose values can reach * the target are inspected: the condition of an IF or of a CASE branch only chooses a value, so a - * Variant path compared with a scalar there loses nothing that is written.

+ * Variant path compared with a scalar there loses nothing that is written. Likewise the later + * inputs of EXCEPT and INTERSECT only decide which rows of the first input stay.

*/ public static void validateNoLossyCoercion(List targetColumns, Plan sourcePlan) { validateNoLossyCoercion(targetColumns, sourcePlan, Optional.empty()); @@ -192,22 +200,55 @@ private static Set traceOutputLineage( private static Set traceRecursiveCte( Plan plan, RecursiveCte recursiveCte, Set requiredExprIds, String targetColumn, TraceContext context) { + // The recursive term reads the previous iteration through the work table, so a column it writes + // can carry the value of another column. Trace those columns as well until no new one is reached. + Map workTableOrdinals = workTableOrdinals(plan); Set externalExprIds = new HashSet<>(); - Set requiredOrdinals = requiredOutputOrdinals(plan, requiredExprIds); - for (int childIndex = 0; childIndex < plan.arity(); childIndex++) { - List childOutputs = recursiveCte.getRegularChildOutput(childIndex); - Set childRequiredExprIds = new HashSet<>(); - for (int ordinal : requiredOrdinals) { - if (ordinal < childOutputs.size()) { - childRequiredExprIds.add(childOutputs.get(ordinal).getExprId()); + Set tracedOrdinals = new HashSet<>(); + Set pendingOrdinals = requiredOutputOrdinals(plan, requiredExprIds); + while (!pendingOrdinals.isEmpty()) { + tracedOrdinals.addAll(pendingOrdinals); + Set childExternalExprIds = new HashSet<>(); + for (int childIndex = 0; childIndex < plan.arity(); childIndex++) { + List childOutputs = recursiveCte.getRegularChildOutput(childIndex); + Set childRequiredExprIds = new HashSet<>(); + for (int ordinal : pendingOrdinals) { + if (ordinal < childOutputs.size()) { + childRequiredExprIds.add(childOutputs.get(ordinal).getExprId()); + } + } + childExternalExprIds.addAll(traceOutputLineage( + plan.child(childIndex), childRequiredExprIds, targetColumn, context)); + } + pendingOrdinals = new HashSet<>(); + for (ExprId exprId : childExternalExprIds) { + Integer ordinal = workTableOrdinals.get(exprId); + if (ordinal == null) { + externalExprIds.add(exprId); + } else if (!tracedOrdinals.contains(ordinal)) { + pendingOrdinals.add(ordinal); } } - externalExprIds.addAll(traceOutputLineage( - plan.child(childIndex), childRequiredExprIds, targetColumn, context)); } return externalExprIds; } + /** Maps each column of the work table the recursive term reads to the CTE output ordinal it holds. */ + private static Map workTableOrdinals(Plan recursiveCte) { + CTEId cteId = ((LogicalRecursiveUnionAnchor) recursiveCte.child(0)).getCteId(); + List workTables = recursiveCte.child(1).collectToList( + node -> node instanceof LogicalWorkTableReference + && ((LogicalWorkTableReference) node).getCteId().equals(cteId)); + Map ordinals = new HashMap<>(); + for (LogicalWorkTableReference workTable : workTables) { + List outputs = workTable.getOutput(); + for (int ordinal = 0; ordinal < outputs.size(); ordinal++) { + ordinals.put(outputs.get(ordinal).getExprId(), ordinal); + } + } + return ordinals; + } + private static Set traceGenerate( LogicalGenerate generate, Set requiredExprIds, String targetColumn, TraceContext context) { @@ -247,7 +288,12 @@ private static Set traceSetOperation( } } - for (int childIndex = 0; childIndex < setOperation.getArity(); childIndex++) { + // EXCEPT and INTERSECT return rows of their first input; the later inputs only decide which of + // those rows stay, so none of their values is written. + int valueInputs = plan instanceof LogicalExcept || plan instanceof LogicalIntersect + ? 1 + : setOperation.getArity(); + for (int childIndex = 0; childIndex < valueInputs; childIndex++) { List childOutputs = setOperation.getRegularChildOutput(childIndex); Set childRequiredExprIds = new HashSet<>(); for (int ordinal : requiredOrdinals) { @@ -275,21 +321,37 @@ private static Set requiredOutputOrdinals(Plan plan, Set requir private static Set traceCteConsumer( LogicalCTEConsumer consumer, Set requiredExprIds, String targetColumn, TraceContext context) { + Set producerExprIds = new HashSet<>(); + for (Map.Entry mapping : consumer.getConsumerToProducerOutputMap().entrySet()) { + if (requiredExprIds.contains(mapping.getKey().getExprId())) { + producerExprIds.add(mapping.getValue().getExprId()); + } + } + // A producer shared by several consumers is traced once per set of requested outputs, so the + // work stays proportional to the plan rather than to the number of paths through it. + Pair> traceKey = Pair.of(consumer.getCteId(), ImmutableSet.copyOf(producerExprIds)); + Set completedTrace = context.completedCteTraces.get(traceKey); + if (completedTrace != null) { + return completedTrace; + } LogicalCTEProducer producer = context.cteProducers.get(consumer.getCteId()); - Optional producerPlan = producer == null - ? context.findAnalyzerCteProducer(consumer) - : Optional.of(producer.child()); + Optional producerPlan; + if (producer == null) { + // A producer taken from the analyzer context is not part of the traced plan, so the + // producers of the WITH clauses nested in it have not been collected yet. + producerPlan = context.findAnalyzerCteProducer(consumer); + producerPlan.ifPresent(context::collectCteProducers); + } else { + producerPlan = Optional.of(producer.child()); + } if (!producerPlan.isPresent() || !context.activeCteIds.add(consumer.getCteId())) { return Collections.emptySet(); } try { - Set producerExprIds = new HashSet<>(); - for (Map.Entry mapping : consumer.getConsumerToProducerOutputMap().entrySet()) { - if (requiredExprIds.contains(mapping.getKey().getExprId())) { - producerExprIds.add(mapping.getValue().getExprId()); - } - } - return traceOutputLineage(producerPlan.get(), producerExprIds, targetColumn, context); + Set externalExprIds = ImmutableSet.copyOf( + traceOutputLineage(producerPlan.get(), producerExprIds, targetColumn, context)); + context.completedCteTraces.put(traceKey, externalExprIds); + return externalExprIds; } finally { context.activeCteIds.remove(consumer.getCteId()); } @@ -420,6 +482,8 @@ private static boolean losesVariantLeaf(DataType sourceType, DataType targetType private static final class TraceContext { private final Map> cteProducers = new HashMap<>(); private final Set activeCteIds = new HashSet<>(); + // Completed producer traces, keyed by CTE and requested producer outputs, with the ExprIds they return. + private final Map>, Set> completedCteTraces = new HashMap<>(); private final Optional analyzerCteContext; private TraceContext(Plan sourcePlan, Optional analyzerCteContext) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java index 973888bad03960..4071622546d9ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java @@ -41,8 +41,12 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalRecursiveUnionProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalWorkTableReference; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.ConnectorComputeVariantType; import org.apache.doris.nereids.types.IntegerType; @@ -54,6 +58,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.List; import java.util.function.Function; @@ -156,6 +161,20 @@ public void testAnalyzedRecursiveUnionAndGenerateLossAreRejected() { } } + @Test + public void testRecursiveCteLossCarriedThroughAnotherColumnIsRejected() { + // The recursive term writes payload from the work table's a, so from the second row on payload holds + // what the anchor's a lost. MUTATION: tracing only the requested column of each term -> accepted. + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), recursiveCtePlan(0))); + Assertions.assertTrue(exception.getMessage().contains("implicitly casts VARIANT to INT"), + exception.getMessage()); + + // The anchor's a never reaches payload when the recursive term writes payload from payload. + VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), recursiveCtePlan(1)); + } + @Test public void testScalarSubqueryLossIsRejected() { LogicalProject lossyQueryPlan = projectExpression(variant -> new If( @@ -225,6 +244,47 @@ public void testAnalyzerCteContextIsUsedBeforeAnchorConstruction() { exception.getMessage()); } + @Test + public void testWithNestedInAnalyzerContextProducerIsTraced() { + // A leading WITH binds its sink before the outer anchors exist, so the outer producer comes from the + // analyzer context and the WITH nested in it is not part of the traced plan. MUTATION: tracing the + // fetched producer without collecting its nested producers -> accepted. + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + Plan nestedWith = PlanChecker.from(connectContext).analyze( + "WITH inner_source AS (SELECT IF(TRUE, parse_to_variant('{\"lost\":true}'), 1) AS payload) " + + "SELECT payload FROM inner_source").getPlan(); + LogicalSubQueryAlias producerPlan = new LogicalSubQueryAlias<>("outer_source", nestedWith); + // Far from the ids the analyzer gave the nested WITH. + CTEId cteId = new CTEId(1000); + CTEContext cteContext = new CTEContext(cteId, producerPlan, new CTEContext()); + cteContext.setAnalyzedPlan(producerPlan); + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + new RelationId(1000), cteId, "outer_source", producerPlan); + LogicalOneRowRelation source = oneRow( + new Alias(new ScalarSubquery(consumer), "payload")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTarget(), source, cteContext)); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + } + + @Test + public void testSharedCteProducerIsTracedOncePerRequestedOutputs() { + // Each CTE reads the previous one twice, so walking every path would trace c0 2^30 times. MUTATION: + // dropping the completed-trace cache -> times out. + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + StringBuilder sql = new StringBuilder("WITH c0 AS (SELECT parse_to_variant('{\"k\":1}') AS payload)"); + for (int level = 1; level <= 30; level++) { + sql.append(String.format(", c%d AS (SELECT payload FROM c%d UNION ALL SELECT payload FROM c%d)", + level, level - 1, level - 1)); + } + sql.append(" SELECT 1, payload FROM c30"); + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql.toString()).getPlan(); + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(30), + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan)); + } + @Test public void testPrimitiveSourceAndExplicitVariantCastAreAccepted() { LogicalOneRowRelation primitiveSource = oneRow( @@ -324,6 +384,32 @@ public void testColumnsReadOnlyByConditionsAreNotTraced() { exception.getMessage()); } + @Test + public void testLaterExceptAndIntersectInputsAreNotWrittenValues() { + // EXCEPT and INTERSECT write rows of their first input; the second input only decides which rows stay, + // so a lossy cast there writes nothing. MUTATION: tracing every set-operation input -> rejected. + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + String lossy = "IF(TRUE, parse_to_variant('{\"lost\":true}'), 1)"; + String numeric = "CAST(1 AS DECIMAL(38, 9))"; + for (String setOperator : ImmutableList.of("EXCEPT", "INTERSECT")) { + Plan membershipOnly = PlanChecker.from(connectContext).analyze( + "SELECT 1, " + numeric + " AS payload " + setOperator + " SELECT 1, " + lossy).getPlan(); + Assertions.assertDoesNotThrow( + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), membershipOnly), + setOperator); + + // The first input is written, so its lossy cast is still rejected. + Plan written = PlanChecker.from(connectContext).analyze( + "SELECT 1, " + lossy + " AS payload " + setOperator + " SELECT 1, " + numeric).getPlan(); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), written), + setOperator); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), + exception.getMessage()); + } + } + @Test public void testRowLevelDmlRoutingColumnsUseMappedSourceOrdinal() { LogicalOneRowRelation input = variantInput(); @@ -378,6 +464,42 @@ public void testLossyCastInNestedVariantTargetIsRejected() { exception.getMessage()); } + /** + * {@code WITH RECURSIVE source(a, payload) AS (SELECT IF(TRUE, CAST(v AS INT), 1), 1 UNION ALL SELECT a, x + * FROM source) SELECT 1, payload FROM source}, shaped as AnalyzeCTE builds it, where x is the work-table + * column at {@code payloadSource}. + */ + private static Plan recursiveCtePlan(int payloadSource) { + CTEId cteId = new CTEId(1); + LogicalOneRowRelation input = variantInput(); + Plan anchor = new LogicalRecursiveUnionAnchor<>(cteId, new LogicalProject<>(ImmutableList.of( + new Alias(new If( + BooleanLiteral.TRUE, + new Cast(input.getOutput().get(0), IntegerType.INSTANCE), + new IntegerLiteral(1)), "a"), + new Alias(new IntegerLiteral(1), "payload")), input)); + LogicalWorkTableReference workTable = new LogicalWorkTableReference( + new RelationId(2), cteId, anchor.getOutput(), ImmutableList.of("source")); + List previous = workTable.getOutput(); + Plan recursive = new LogicalRecursiveUnionProducer<>("source", new LogicalProject<>(ImmutableList.of( + new Alias(previous.get(0), "a"), + new Alias(previous.get(payloadSource), "payload")), workTable)); + Plan union = new LogicalRecursiveUnion<>( + "source", + Qualifier.ALL, + ImmutableList.of( + new SlotReference("a", IntegerType.INSTANCE), + new SlotReference("payload", IntegerType.INSTANCE)), + ImmutableList.of(slotReferences(anchor), slotReferences(recursive)), + ImmutableList.of(anchor, recursive)); + return new LogicalProject<>(ImmutableList.of( + new Alias(new IntegerLiteral(1), "id"), union.getOutput().get(1)), union); + } + + private static List slotReferences(Plan plan) { + return plan.getOutput().stream().map(SlotReference.class::cast).collect(ImmutableList.toImmutableList()); + } + private static LogicalUnion lossyUnionPlan() { LogicalProject left = projectExpression( variant -> new Cast(variant, IntegerType.INSTANCE)); From b45604b510a3d01246670c93bde39a62a69a3278 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Oct 2026 22:41:46 +0100 Subject: [PATCH 3/4] [fix](fe) Leave window keys and aggregate ORDER BY out of the Variant write check The Variant write validation searched every child of a written expression except the conditions of IF and CASE. The partition and order keys of a window and the ORDER BY of an aggregate function such as GROUP_CONCAT only arrange the rows whose values are used, yet a lossy cast there rejected the statement: INSERT INTO t(payload) SELECT ROW_NUMBER() OVER (ORDER BY IF(TRUE, v, 1)) FROM src failed, although only row numbers are written. A window now contributes only its function to the values checked, and the ORDER BY of an aggregate function nothing, so the columns read only by those keys are not traced either. A lossy cast in an argument of the function is still rejected. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../datasource/VariantWritePlanValidator.java | 16 +++++++-- .../VariantWritePlanValidatorTest.java | 35 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java index 71dd5a4aba459d..7cac5009e17e0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java @@ -26,10 +26,12 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.RecursiveCte; @@ -74,7 +76,9 @@ private VariantWritePlanValidator() { * intentional user conversion and are not rejected. Only the expressions whose values can reach * the target are inspected: the condition of an IF or of a CASE branch only chooses a value, so a * Variant path compared with a scalar there loses nothing that is written. Likewise the later - * inputs of EXCEPT and INTERSECT only decide which rows of the first input stay.

+ * inputs of EXCEPT and INTERSECT only decide which rows of the first input stay, and the partition + * and order keys of a window and the ORDER BY of an aggregate function only arrange the rows whose + * values are used.

*/ public static void validateNoLossyCoercion(List targetColumns, Plan sourcePlan) { validateNoLossyCoercion(targetColumns, sourcePlan, Optional.empty()); @@ -399,7 +403,9 @@ private static Set traceScalarSubquery( /** * The nodes of {@code expression} whose values can reach its result, in pre-order: every node - * except those under the condition of an IF or of a CASE branch, which only chooses a value. + * except those under the condition of an IF or of a CASE branch, which only chooses a value, and + * those under the partition and order keys of a window or the ORDER BY of an aggregate function + * such as GROUP_CONCAT, which only arrange the rows whose values are used. */ private static List valueNodes(Expression expression) { List nodes = new ArrayList<>(); @@ -414,9 +420,13 @@ private static void collectValueNodes(Expression expression, List no collectValueNodes(((If) expression).getFalseValue(), nodes); } else if (expression instanceof WhenClause) { collectValueNodes(((WhenClause) expression).getResult(), nodes); + } else if (expression instanceof WindowExpression) { + collectValueNodes(((WindowExpression) expression).getFunction(), nodes); } else { for (Expression child : expression.children()) { - collectValueNodes(child, nodes); + if (!(child instanceof OrderExpression)) { + collectValueNodes(child, nodes); + } } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java index 4071622546d9ae..dd8792363221a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java @@ -285,6 +285,41 @@ public void testSharedCteProducerIsTracedOncePerRequestedOutputs() { () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan)); } + @Test + public void testWindowAndAggregateOrderingKeysAreNotWrittenValues() { + // The partition and order keys of a window and the ORDER BY of GROUP_CONCAT only arrange the rows whose + // values are used, so a lossy cast there, or in a column read only by such a key, writes nothing. A lossy + // cast in an argument of the function is written and still rejected. MUTATION: tracing the keys of a + // window, or the ORDER BY of an aggregate function -> rejected. + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + String source = "(SELECT 1 AS id, parse_to_variant('{\"k\":1}') AS v) source"; + List keyedOnly = ImmutableList.of( + "SELECT 1, ROW_NUMBER() OVER (ORDER BY IF(TRUE, v, 1)) FROM " + source, + "SELECT 1, COUNT(*) OVER (PARTITION BY IF(TRUE, v, 1)) FROM " + source, + "SELECT 1, GROUP_CONCAT('x' ORDER BY IF(TRUE, v, 1)) FROM " + source, + "SELECT 1, ROW_NUMBER() OVER (ORDER BY k) FROM (SELECT IF(TRUE, v, 1) AS k FROM " + source + + ") keyed"); + for (String sql : keyedOnly) { + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql).getPlan(); + Assertions.assertDoesNotThrow( + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan), + sql); + } + + List written = ImmutableList.of( + "SELECT 1, FIRST_VALUE(IF(TRUE, v, 1)) OVER (ORDER BY id) FROM " + source, + "SELECT 1, GROUP_CONCAT(CAST(IF(TRUE, v, 1) AS STRING) ORDER BY id) FROM " + source); + for (String sql : written) { + Plan analyzedPlan = PlanChecker.from(connectContext).analyze(sql).getPlan(); + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan), + sql); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), + exception.getMessage()); + } + } + @Test public void testPrimitiveSourceAndExplicitVariantCastAreAccepted() { LogicalOneRowRelation primitiveSource = oneRow( From 7e9ef143a837fad2e0ef00062bbd7f3c6c244efb Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Oct 2026 22:41:46 +0100 Subject: [PATCH 4/4] [fix](fe) Trace each output of a shared CTE producer once in the Variant write check The completed traces of a shared CTE producer were cached by the set of outputs a consumer requested, so consumers that requested different sets still traced the producer once per set, and the number of such sets can double at each level: when each CTE reads the previous one once for v0 alone and once for v0 together with another column, 24 levels request 2^24 sets of the first one. The traces are now cached per producer output, and a consumer combines those of the outputs it requests, so a producer is traced at most once for each of its outputs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../datasource/VariantWritePlanValidator.java | 25 +++++++---- .../VariantWritePlanValidatorTest.java | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java index 7cac5009e17e0e..36a02ca14f1a6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/VariantWritePlanValidator.java @@ -325,15 +325,22 @@ private static Set requiredOutputOrdinals(Plan plan, Set requir private static Set traceCteConsumer( LogicalCTEConsumer consumer, Set requiredExprIds, String targetColumn, TraceContext context) { - Set producerExprIds = new HashSet<>(); + // A producer shared by several consumers is traced once per output, whichever sets of its outputs + // the consumers request, so the work stays proportional to the plan rather than to the number of + // paths through it or of the sets of outputs requested along them. + Set externalExprIds = new HashSet<>(); for (Map.Entry mapping : consumer.getConsumerToProducerOutputMap().entrySet()) { if (requiredExprIds.contains(mapping.getKey().getExprId())) { - producerExprIds.add(mapping.getValue().getExprId()); + externalExprIds.addAll(traceCteOutput( + consumer, mapping.getValue().getExprId(), targetColumn, context)); } } - // A producer shared by several consumers is traced once per set of requested outputs, so the - // work stays proportional to the plan rather than to the number of paths through it. - Pair> traceKey = Pair.of(consumer.getCteId(), ImmutableSet.copyOf(producerExprIds)); + return externalExprIds; + } + + private static Set traceCteOutput( + LogicalCTEConsumer consumer, ExprId producerExprId, String targetColumn, TraceContext context) { + Pair traceKey = Pair.of(consumer.getCteId(), producerExprId); Set completedTrace = context.completedCteTraces.get(traceKey); if (completedTrace != null) { return completedTrace; @@ -352,8 +359,8 @@ private static Set traceCteConsumer( return Collections.emptySet(); } try { - Set externalExprIds = ImmutableSet.copyOf( - traceOutputLineage(producerPlan.get(), producerExprIds, targetColumn, context)); + Set externalExprIds = ImmutableSet.copyOf(traceOutputLineage( + producerPlan.get(), Collections.singleton(producerExprId), targetColumn, context)); context.completedCteTraces.put(traceKey, externalExprIds); return externalExprIds; } finally { @@ -492,8 +499,8 @@ private static boolean losesVariantLeaf(DataType sourceType, DataType targetType private static final class TraceContext { private final Map> cteProducers = new HashMap<>(); private final Set activeCteIds = new HashSet<>(); - // Completed producer traces, keyed by CTE and requested producer outputs, with the ExprIds they return. - private final Map>, Set> completedCteTraces = new HashMap<>(); + // Completed traces of producer outputs, keyed by CTE and output, with the ExprIds they return. + private final Map, Set> completedCteTraces = new HashMap<>(); private final Optional analyzerCteContext; private TraceContext(Plan sourcePlan, Optional analyzerCteContext) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java index dd8792363221a1..863c52282581e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/VariantWritePlanValidatorTest.java @@ -285,6 +285,25 @@ public void testSharedCteProducerIsTracedOncePerRequestedOutputs() { () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), analyzedPlan)); } + @Test + public void testCteOutputsRequestedInManySetsAreTracedOnceEach() { + // c(i) reads c(i - 1) twice, for v0 alone and for v0 together with v(i), so the sets of outputs requested + // from c(24 - k) double with k: a cache keyed by the set of requested outputs would trace c0 2^24 times. + // Each output is traced once, and a loss carried by one of them only is still found. MUTATION: caching + // the traces per set of requested outputs -> times out or runs out of memory. + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + Plan lossless = PlanChecker.from(connectContext) + .analyze(cteChainRequestingDoublingSets(24, "parse_to_variant('{\"k\":1}')")).getPlan(); + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(30), + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), lossless)); + + Plan lossy = PlanChecker.from(connectContext).analyze(cteChainRequestingDoublingSets( + 24, "CAST(IF(TRUE, parse_to_variant('{\"lost\":true}'), 1) AS VARIANT)")).getPlan(); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> VariantWritePlanValidator.validateNoLossyCoercion(variantTargetWithId(), lossy)); + Assertions.assertTrue(exception.getMessage().contains("input column 'payload'"), exception.getMessage()); + } + @Test public void testWindowAndAggregateOrderingKeysAreNotWrittenValues() { // The partition and order keys of a window and the ORDER BY of GROUP_CONCAT only arrange the rows whose @@ -531,6 +550,29 @@ private static Plan recursiveCtePlan(int payloadSource) { new Alias(new IntegerLiteral(1), "id"), union.getOutput().get(1)), union); } + /** + * WITH c0 (v0 ... v{levels}), then c(i) = SELECT COALESCE(v0, v(i)) AS v0, v1 ... FROM c(i - 1) UNION ALL + * SELECT v0, v1 ... FROM c(i - 1), and SELECT 1, v0 FROM c{levels}. The last column of c0 is {@code lastValue} + * and the others a Variant literal. + */ + private static String cteChainRequestingDoublingSets(int levels, String lastValue) { + StringBuilder columns = new StringBuilder(); + for (int column = 1; column <= levels; column++) { + columns.append(", v").append(column); + } + StringBuilder sql = new StringBuilder("WITH c0 AS (SELECT "); + for (int column = 0; column < levels; column++) { + sql.append("parse_to_variant('{\"k\":1}') AS v").append(column).append(", "); + } + sql.append(lastValue).append(" AS v").append(levels).append(')'); + for (int level = 1; level <= levels; level++) { + sql.append(String.format( + ", c%d AS (SELECT COALESCE(v0, v%d) AS v0%s FROM c%d UNION ALL SELECT v0%s FROM c%d)", + level, level, columns, level - 1, columns, level - 1)); + } + return sql.append(" SELECT 1, v0 FROM c").append(levels).toString(); + } + private static List slotReferences(Plan plan) { return plan.getOutput().stream().map(SlotReference.class::cast).collect(ImmutableList.toImmutableList()); }