From 7634b4634d8b54053cd886c5a1dfe23ff9dc3e0b Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 12 Oct 2026 08:13:12 +0100 Subject: [PATCH] [test](paimon) Port the branch-4.x Paimon Variant JNI read and access path tests ### What problem does this PR solve? Issue Number: None Related PR: #66334, #66547, #66573, #67395, #66399 Problem Summary: branch-4.x added Paimon VARIANT reads through the JNI reader (#66334), Variant sub-path access paths for that reader (#66547) and nested type pruning for it (#66573). Their production code reached master with #67395 and #66399, but several of their tests did not: - BE: JniDataBridgeTest (Variant JNI type names, and filling a nullable Variant V2 column from Paimon's encoded layout with a deduplicated metadata dictionary); PaimonJniReaderTest's check that variant_access_path.. follows the projected column position; and ParquetSchemaTest's positive case for OPTIONAL Paimon shredding wrappers on an object field and an array element, which replaces the metadata-only case. - BE Java: six layout assertions in VectorColumnVariantTest, PaimonColumnValueTest#testGetVariantEncodedBytes, PaimonJniScannerTest#testVariantAccessPathsStayAlignedWithRequiredFields, and four PaimonVariantProjectionTest cases, including the guard against the misaligned Variant null bitmap of Paimon 1.4.2. - FE: no test pinned that the Paimon connector declares SUPPORTS_NESTED_COLUMN_PRUNE but not SUPPORTS_FIELD_ID_ACCESS_PATH. PaimonConnectorMetadataMvccTest now does. - Regression: test_paimon_catalog_variant gains the four JNI query blocks and three EXPLAIN checks of #66547 (access paths payload.name and payload.profile.city under JNI, payload.name under the native reader). The variant_shredded fixture in run13.sql gets the profile.city shredding and the tags values of #66547, which two of the blocks need. Paimon 1.3.1 in the docker environment writes that schema. In the branch-4.x suite, one explain block has both contains and check, and the framework skips contains when check is set. The access path checks here are inside the check closures. Tests for code that master does not have, such as the enable_variant_v2 switch, are not ported. ### Release note None ### Check List (For Author) - Test: Regression test / Unit Test - BE UT: JniDataBridgeTest, PaimonJniReaderTest and ParquetSchemaTest pass. - Java UT: VectorColumnVariantTest, PaimonColumnValueTest, PaimonJniScannerTest and PaimonVariantProjectionTest pass. - FE UT: PaimonConnectorMetadataMvccTest passes. - Regression: test_paimon_catalog_variant (.out regenerated by the script; the 17 existing blocks are unchanged and the 4 new ones match branch-4.x) and test_paimon_jni_complex_column_pruning pass against the Iceberg docker environment with the updated fixture. - Behavior changed: No - Does this need documentation: No Co-authored-by: zhangstar333 Co-Authored-By: Claude Opus 5.5 (1M context) --- be/test/format/jni/jni_data_bridge_test.cpp | 89 +++++++++++++ .../format_v2/jni/paimon_jni_reader_test.cpp | 29 ++++- .../format_v2/parquet/parquet_schema_test.cpp | 36 ++++-- .../paimon/run13.sql | 6 +- .../jni/spi/vec/VectorColumnVariantTest.java | 12 ++ .../doris/paimon/PaimonColumnValueTest.java | 15 +++ .../doris/paimon/PaimonJniScannerTest.java | 14 +++ .../paimon/PaimonVariantProjectionTest.java | 119 ++++++++++++++++++ .../PaimonConnectorMetadataMvccTest.java | 18 +++ .../paimon/test_paimon_catalog_variant.out | 16 +++ .../paimon/test_paimon_catalog_variant.groovy | 73 +++++++++++ 11 files changed, 410 insertions(+), 17 deletions(-) create mode 100644 be/test/format/jni/jni_data_bridge_test.cpp diff --git a/be/test/format/jni/jni_data_bridge_test.cpp b/be/test/format/jni/jni_data_bridge_test.cpp new file mode 100644 index 00000000000000..d6418c0b97e28d --- /dev/null +++ b/be/test/format/jni/jni_data_bridge_test.cpp @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/jni/jni_data_bridge.h" + +#include + +#include +#include + +#include "core/column/column_nullable.h" +#include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_variant_v2.h" + +namespace doris { +namespace { + +template +long address_of(T* pointer) { + return reinterpret_cast(pointer); +} + +TEST(JniDataBridgeTest, VariantTypeUsesVariantJniName) { + auto variant_type = std::make_shared(); + auto nullable_variant_type = make_nullable(variant_type); + + EXPECT_EQ(JniDataBridge::get_jni_type(variant_type), "variant"); + EXPECT_EQ(JniDataBridge::get_jni_type(nullable_variant_type), "variant"); + EXPECT_EQ(JniDataBridge::get_jni_type_with_different_string(variant_type), "variant"); + EXPECT_EQ(JniDataBridge::get_jni_type_with_different_string(nullable_variant_type), "variant"); +} + +TEST(JniDataBridgeTest, FillNullableVariantV2FromEncodedPaimonLayout) { + std::array null_map {false, true, false}; + // Paimon metadata contains an empty dictionary and a one-key dictionary for "a". + std::array metadata_offsets {0, 3, 8}; + std::array metadata_bytes {char {0x01}, char {0x00}, char {0x00}, char {0x01}, + char {0x01}, char {0x00}, char {0x01}, 'a'}; + std::array metadata_ids {0, 0, 1}; + // Primitive values are true, Variant null (the SQL NULL placeholder), and false. + std::array value_offsets {0, 1, 2, 3}; + std::array value_bytes {char {0x04}, char {0x00}, char {0x08}}; + + std::array meta { + address_of(null_map.data()), 2, + address_of(metadata_offsets.data()), address_of(metadata_bytes.data()), + address_of(metadata_ids.data()), address_of(value_offsets.data()), + address_of(value_bytes.data()), + }; + JniDataBridge::TableMetaAddress meta_address(address_of(meta.data())); + + DataTypePtr data_type = make_nullable(std::make_shared()); + ColumnPtr column = data_type->create_column(); + const Status status = JniDataBridge::fill_column(meta_address, column, data_type, 3); + ASSERT_TRUE(status.ok()) << status; + + const auto& nullable_column = assert_cast(*column); + EXPECT_EQ(nullable_column.get_null_map_data(), NullMap({0, 1, 0})); + + const auto& variant_column = + assert_cast(nullable_column.get_nested_column()); + ASSERT_EQ(variant_column.size(), 3); + ASSERT_FALSE(variant_column.is_typed()); + const auto view = variant_column.read_view(); + EXPECT_EQ(view.metadata_count(), 2); + EXPECT_EQ(view.metadata_id_at(0), view.metadata_id_at(1)); + EXPECT_NE(view.metadata_id_at(0), view.metadata_id_at(2)); + EXPECT_TRUE(variant_column.get_value_ref(0).get_bool()); + EXPECT_TRUE(variant_column.get_value_ref(1).is_null()); + EXPECT_FALSE(variant_column.get_value_ref(2).get_bool()); +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/jni/paimon_jni_reader_test.cpp b/be/test/format_v2/jni/paimon_jni_reader_test.cpp index b97e6e6126b278..8ce9e025a13f9a 100644 --- a/be/test/format_v2/jni/paimon_jni_reader_test.cpp +++ b/be/test/format_v2/jni/paimon_jni_reader_test.cpp @@ -24,6 +24,7 @@ #include #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_variant_v2.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "runtime/runtime_state.h" @@ -50,9 +51,10 @@ TFileScanRangeParams make_scan_params() { } Status init_reader(PaimonJniReader* reader, TFileScanRangeParams* scan_params, - RuntimeState* runtime_state = nullptr) { + RuntimeState* runtime_state = nullptr, + std::vector projected_columns = {}) { return reader->init({ - .projected_columns = {}, + .projected_columns = std::move(projected_columns), .conjuncts = {}, .format = FileFormat::JNI, .scan_params = scan_params, @@ -68,6 +70,29 @@ Status build_params(PaimonJniReader* reader, const TFileRangeDesc& range, return reader->build_scanner_params(params); } +TEST(PaimonJniReaderTest, PublishesVariantAccessPathsByProjectedColumnPosition) { + auto range = make_paimon_jni_range(); + range.table_format_params.paimon_params.__set_paimon_predicate("serialized-predicate"); + auto scan_params = make_scan_params(); + + ColumnDefinition id; + id.name = "id"; + id.type = std::make_shared(); + ColumnDefinition payload; + payload.name = "payload"; + payload.type = std::make_shared(); + payload.variant_access_paths = {{"name"}, {"profile", "city"}}; + + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, nullptr, {id, payload}).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_FALSE(params.contains("variant_access_path.0.0")); + EXPECT_EQ(params.at("variant_access_path.1.0"), "$bmFtZQ=="); + EXPECT_EQ(params.at("variant_access_path.1.1"), "$cHJvZmlsZQ==,$Y2l0eQ=="); +} + TEST(PaimonJniReaderTest, UsesScanLevelPredicateBeforeLegacySplitPredicate) { auto range = make_paimon_jni_range(); range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate"); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index d57c15fedfe239..f7f96ea2c7e624 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -291,19 +291,31 @@ TEST(ParquetSchemaTest, AppliesTableFormatVariantOverrideToUnannotatedGroup) { EXPECT_NE(fields[0]->variant_physical_type, nullptr); } -TEST(ParquetSchemaTest, AppliesPaimonShreddedVariantOverrideWithOptionalMetadata) { - auto schema = shredded_object_variant_schema(); - schema[1].__isset.logicalType = false; - schema[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); - NativeFieldDescriptor descriptor; - ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok()); +TEST(ParquetSchemaTest, AppliesPaimonShreddedVariantOverrideWithOptionalFields) { + const auto apply_override = [](std::vector schema) { + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok()); - std::vector> fields; - ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; - const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(fields.size(), 1); + EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT); + }; + + auto object_schema = shredded_object_variant_schema(); + object_schema[1].__isset.logicalType = false; + object_schema[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + object_schema[5].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + apply_override(std::move(object_schema)); + + auto array_schema = shredded_array_variant_schema(true, true); + array_schema[1].__isset.logicalType = false; + array_schema[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + array_schema[6].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + apply_override(std::move(array_schema)); } TEST(ParquetSchemaTest, AppliesPaimonFallbackOnlyArrayWithRequiredValue) { diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql index 0f5804f610b6fe..ab1db3da52737d 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql @@ -25,12 +25,12 @@ create table variant_shredded ( partitioned by (event_date) tblproperties ( 'file.format' = 'parquet', - 'parquet.variant.shreddingSchema' = '{"type":"ROW","fields":[{"name":"payload","type":{"type":"ROW","fields":[{"name":"name","type":"STRING"},{"name":"age","type":"INT"}]}}]}' + 'parquet.variant.shreddingSchema' = '{"type":"ROW","fields":[{"name":"payload","type":{"type":"ROW","fields":[{"name":"name","type":"STRING"},{"name":"age","type":"INT"},{"name":"profile","type":{"type":"ROW","fields":[{"name":"city","type":"STRING"}]}}]}}]}' ); insert into variant_shredded values - (1, date '2026-06-01', parse_json('{"name":"alice","age":18,"extra":"shredded"}')), - (2, date '2026-06-01', parse_json('{"name":"bob","age":30}')); + (1, date '2026-06-01', parse_json('{"name":"alice","age":18,"profile":{"city":"beijing"},"tags":["flink","paimon"],"extra":"shredded"}')), + (2, date '2026-06-01', parse_json('{"name":"bob","age":30,"profile":{"city":"shanghai"},"tags":["doris"]}')); drop table if exists variant_mixed_us; create table variant_mixed_us ( diff --git a/fe/be-java-extensions/jni-spi/src/test/java/org/apache/doris/jni/spi/vec/VectorColumnVariantTest.java b/fe/be-java-extensions/jni-spi/src/test/java/org/apache/doris/jni/spi/vec/VectorColumnVariantTest.java index 8f85a9e4f7400d..1c90a95a25433a 100644 --- a/fe/be-java-extensions/jni-spi/src/test/java/org/apache/doris/jni/spi/vec/VectorColumnVariantTest.java +++ b/fe/be-java-extensions/jni-spi/src/test/java/org/apache/doris/jni/spi/vec/VectorColumnVariantTest.java @@ -46,13 +46,19 @@ public void testVariantTypeAndEncodedLayout() { column.appendVariant(ONE_KEY_METADATA, new byte[] {8}); long meta = table.getMetaAddress(); Assertions.assertEquals(3L, OffHeap.getLong(null, meta)); + Assertions.assertArrayEquals(new boolean[] {false, false, false}, + OffHeap.getBoolean(null, OffHeap.getLong(null, meta + 8), 3)); Assertions.assertEquals(2L, OffHeap.getLong(null, meta + 16)); Assertions.assertArrayEquals(new int[] {0, 3, 8}, OffHeap.getInt(null, OffHeap.getLong(null, meta + 24), 3)); + Assertions.assertArrayEquals(new byte[] {1, 0, 0, 1, 1, 0, 1, 'a'}, + OffHeap.getByte(null, OffHeap.getLong(null, meta + 32), 8)); Assertions.assertArrayEquals(new int[] {0, 0, 1}, OffHeap.getInt(null, OffHeap.getLong(null, meta + 40), 3)); Assertions.assertArrayEquals(new int[] {0, 1, 2, 3}, OffHeap.getInt(null, OffHeap.getLong(null, meta + 48), 4)); + Assertions.assertArrayEquals(new byte[] {0, 4, 8}, + OffHeap.getByte(null, OffHeap.getLong(null, meta + 56), 3)); } finally { table.close(); } @@ -71,6 +77,10 @@ public void testSqlNullUsesValidVariantNullPlaceholder() { Assertions.assertArrayEquals(new boolean[] {false, true}, OffHeap.getBoolean(null, OffHeap.getLong(null, meta + 8), 2)); Assertions.assertEquals(1L, OffHeap.getLong(null, meta + 16)); + Assertions.assertArrayEquals(new int[] {0, 0}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 40), 2)); + Assertions.assertArrayEquals(new int[] {0, 1, 2}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 48), 3)); Assertions.assertArrayEquals(new byte[] {4, 0}, OffHeap.getByte(null, OffHeap.getLong(null, meta + 56), 2)); } finally { @@ -92,6 +102,8 @@ public void testResetRebuildsMetadataDictionary() { Assertions.assertEquals(1L, OffHeap.getLong(null, meta + 16)); Assertions.assertArrayEquals(new int[] {0, 5}, OffHeap.getInt(null, OffHeap.getLong(null, meta + 24), 2)); + Assertions.assertArrayEquals(new int[] {0}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 40), 1)); } finally { table.close(); } diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java index a82e412be1dae0..f8c8a34ddb26ce 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataType; @@ -39,6 +40,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VariantType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -97,6 +99,19 @@ public void testUnpackCompleteStruct() { Assertions.assertEquals(100L, values.get(2).getLong()); } + @Test + public void testGetVariantEncodedBytes() { + GenericVariant variant = GenericVariant.fromJson("{\"a\":[1,null]}"); + PaimonColumnValue columnValue = new PaimonColumnValue( + GenericRow.of(variant), 0, ColumnType.parseType("v", "variant"), new VariantType(), "UTC"); + + Assertions.assertArrayEquals(variant.metadata(), columnValue.getVariantMetadata()); + Assertions.assertArrayEquals(variant.value(), columnValue.getVariantValue()); + Assertions.assertEquals( + ColumnType.Type.VARIANT, + PaimonTypeUtils.fromPaimonType("v", new VariantType()).getType()); + } + @Test public void testReuseArrayElementsAcrossRows() { ArrayType paimonArrayType = new ArrayType(new IntType()); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 0f53db77db7111..4cdc78dea2e71a 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -63,6 +63,7 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -571,6 +572,19 @@ public void testConstructorDecodesDelimiterSafeNestedFieldNamesAndTypes() { structType.getChildNames()); } + @Test + public void testVariantAccessPathsStayAlignedWithRequiredFields() { + Map params = createBaseParams(); + params.put("variant_access_path.1.0", encodeFields("name")); + params.put("variant_access_path.1.1", encodeFields("profile", "city")); + + List>> paths = PaimonJniScanner.variantAccessPathsByColumn(params, 3); + Assertions.assertTrue(paths.get(0).isEmpty()); + Assertions.assertEquals(Collections.singletonList("name"), paths.get(1).get(0)); + Assertions.assertEquals(Arrays.asList("profile", "city"), paths.get(1).get(1)); + Assertions.assertTrue(paths.get(2).isEmpty()); + } + @Test public void testConstructorDistinguishesEmptyIdentifierFromEmptyProjection() { Map oneEmptyField = createBaseParams(); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java index 7db1c2fa18ccb4..16167f4a026057 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java @@ -18,11 +18,16 @@ package org.apache.doris.paimon; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.columnar.ColumnVector; +import org.apache.paimon.data.columnar.heap.HeapBytesVector; +import org.apache.paimon.data.columnar.heap.HeapRowVector; import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.data.variant.Variant; +import org.apache.paimon.data.variant.VariantMetadataUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; public class PaimonVariantProjectionTest { @@ -40,4 +45,118 @@ public void missingProjectedPathMaterializesAnEmptyObject() { Assertions.assertArrayEquals(expected.value(), actual.value()); Assertions.assertArrayEquals(expected.metadata(), actual.metadata()); } + + @Test + public void testBuildsMetadataMarkedReadTypeAndPartialVariant() { + PaimonVariantProjection projection = PaimonVariantProjection.create( + Arrays.asList( + Collections.singletonList("name"), + Arrays.asList("profile", "city"), + Collections.singletonList("missing")), + "Asia/Shanghai"); + + Assertions.assertNotNull(projection); + Assertions.assertEquals("$.name", VariantMetadataUtils.path( + projection.readType().getFields().get(0).description())); + Assertions.assertEquals("$.profile.city", VariantMetadataUtils.path( + projection.readType().getFields().get(1).description())); + Assertions.assertFalse(VariantMetadataUtils.failOnError( + projection.readType().getFields().get(1).description())); + + GenericRow extracted = projectedRecord( + GenericVariant.fromJson("\"alice\""), + GenericVariant.fromJson("\"beijing\""), + null); + Variant result = projection.materialize(extracted, 0); + Assertions.assertEquals( + "{\"name\":\"alice\",\"profile\":{\"city\":\"beijing\"}}", + result.toJson()); + } + + @Test + public void testPreservesJsonNullButOmitsMissingPath() { + PaimonVariantProjection projection = PaimonVariantProjection.create( + Arrays.asList( + Collections.singletonList("present"), + Collections.singletonList("missing")), + "UTC"); + + Variant result = projection.materialize( + projectedRecord(GenericVariant.fromJson("null"), null), 0); + Assertions.assertEquals("{\"present\":null}", result.toJson()); + } + + @Test + public void testIgnoresMisalignedPaimonVariantNullBitmap() { + PaimonVariantProjection projection = PaimonVariantProjection.create( + Collections.singletonList(Collections.singletonList("name")), "UTC"); + + GenericVariant alice = GenericVariant.fromJson("\"alice\""); + GenericVariant bob = GenericVariant.fromJson("\"bob\""); + HeapBytesVector values = new HeapBytesVector(3); + HeapBytesVector metadata = new HeapBytesVector(3); + appendVariant(values, metadata, alice); + appendVariant(values, metadata, bob); + + HeapRowVector variants = new HeapRowVector(3, values, metadata); + // Paimon 1.4.2 does not advance this row vector for non-null Variants. Appending the + // missing third value consequently marks row 0 null even though its binary children hold + // Alice; the binary children themselves still have the correct row alignment. + variants.appendNull(); + HeapRowVector extractedRows = new HeapRowVector(3, variants); + extractedRows.appendRow(); + extractedRows.appendRow(); + extractedRows.appendRow(); + + Assertions.assertEquals( + "{\"name\":\"alice\"}", + projection.materialize(GenericRow.of(extractedRows.getRow(0)), 0).toJson()); + Assertions.assertEquals( + "{\"name\":\"bob\"}", + projection.materialize(GenericRow.of(extractedRows.getRow(1)), 0).toJson()); + Assertions.assertEquals( + "{}", projection.materialize(GenericRow.of(extractedRows.getRow(2)), 0).toJson()); + } + + @Test + public void testFallsBackForAmbiguousOrUnsupportedPaths() { + Assertions.assertNull(PaimonVariantProjection.create( + Collections.singletonList(Collections.singletonList("1")), "UTC")); + Assertions.assertNull(PaimonVariantProjection.create( + Collections.singletonList(Collections.singletonList("a.b")), "UTC")); + Assertions.assertNull(PaimonVariantProjection.create( + Collections.singletonList(Collections.singletonList("a;b")), "UTC")); + Assertions.assertNull(PaimonVariantProjection.create( + Arrays.asList( + Collections.singletonList("profile"), + Arrays.asList("profile", "city")), + "UTC")); + } + + private static void appendVariant( + HeapBytesVector values, HeapBytesVector metadata, GenericVariant variant) { + values.appendByteArray(variant.value(), 0, variant.value().length); + metadata.appendByteArray(variant.metadata(), 0, variant.metadata().length); + } + + private static GenericRow projectedRecord(Variant... extractedValues) { + ColumnVector[] fields = new ColumnVector[extractedValues.length]; + for (int i = 0; i < extractedValues.length; i++) { + HeapBytesVector values = new HeapBytesVector(1); + HeapBytesVector metadata = new HeapBytesVector(1); + HeapRowVector variant = new HeapRowVector(1, values, metadata); + if (extractedValues[i] == null) { + variant.appendNull(); + } else { + GenericVariant value = new GenericVariant( + extractedValues[i].value(), extractedValues[i].metadata()); + appendVariant(values, metadata, value); + variant.appendRow(); + } + fields[i] = variant; + } + HeapRowVector extracted = new HeapRowVector(1, fields); + extracted.appendRow(); + return GenericRow.of(extracted.getRow(0)); + } } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java index e8d7f33ea8f4ca..813ee4083d7c9b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java @@ -1230,4 +1230,22 @@ public void connectorDeclaresShowCreateDdlCapability() { "paimon must declare SUPPORTS_SHOW_CREATE_DDL so SHOW CREATE TABLE keeps rendering " + "LOCATION/PROPERTIES"); } + + @Test + public void connectorDeclaresNestedColumnPruneButNotFieldIdAccessPath() { + ConnectorContext ctx = new RecordingConnectorContext(); + Set caps = new PaimonConnector(Collections.emptyMap(), ctx).getCapabilities(); + + // WHY: fe-core prunes nested types and plans Variant sub-path access paths for a table only when its + // connector declares SUPPORTS_NESTED_COLUMN_PRUNE. The paimon JNI scanner turns them into a paimon read + // type and Variant projections, and the native parquet/orc path resolves them by name. MUTATION: + // dropping SUPPORTS_NESTED_COLUMN_PRUNE -> paimon scans read whole nested and Variant columns -> red. + Assertions.assertTrue(caps.contains(ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE), + "paimon must declare SUPPORTS_NESTED_COLUMN_PRUNE"); + // Paimon carries no field id on the Doris column tree, so rewriting the access paths to ids would make + // every segment "-1", which BE matches neither as an id nor as a name, and the pruned leaves would read + // back NULL. MUTATION: declaring SUPPORTS_FIELD_ID_ACCESS_PATH on paimon -> this assertion red. + Assertions.assertFalse(caps.contains(ConnectorCapability.SUPPORTS_FIELD_ID_ACCESS_PATH), + "paimon must NOT declare SUPPORTS_FIELD_ID_ACCESS_PATH (its access paths are by name)"); + } } diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out index 830e466af48556..e95b979b4fa3ea 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out @@ -24,6 +24,21 @@ payload variant + def nativeSplits = explainString =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + return explainString.contains("all access paths: [payload.name]") + && nativeSplits.find() + && nativeSplits.group(1).toInteger() == 0 + && nativeSplits.group(2).toInteger() > 0 + } + } + // Keep the complete DESC shape so the external column's Extra metadata remains covered. order_qt_desc """desc variant_smoke""" @@ -93,6 +104,57 @@ suite("test_paimon_catalog_variant", "p0,external,doris,external_docker,external order by id """ + // Exercise Paimon's metadata-marked read type against a physically shredded file. The + // variant_smoke cases above read unshredded files. + order_qt_jni_shredded_projection """ + select id, + cast(payload['name'] as string), + cast(payload['age'] as int) + from variant_shredded + where cast(payload['age'] as int) >= 20 + order by id + """ + + // profile.city is physically shredded. This covers the complete FE -> BE -> JNI -> + // Paimon readType path for a nested object projection, beyond the Java projection UT. + explain { + sql "select id, cast(payload['profile']['city'] as string) from variant_shredded" + check { explainString -> + def nativeSplits = explainString =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + return explainString.contains("all access paths: [payload.profile.city]") + && nativeSplits.find() + && nativeSplits.group(1).toInteger() == 0 + && nativeSplits.group(2).toInteger() > 0 + } + } + + order_qt_jni_nested_shredded_path """ + select id, cast(payload['profile']['city'] as string) + from variant_shredded + order by id + """ + + // Doris Variant array indexes are one-based. The numeric path segment is intentionally + // unsupported by Paimon's metadata projection, so it must make the whole Variant column + // fall back even though payload.name alone is projectable. + order_qt_jni_unsupported_array_path_fallback """ + select id, + cast(payload['name'] as string), + cast(payload['tags'][1] as string) + from variant_shredded + order by id + """ + + // A table can contain files written before and after shredding was enabled. Paimon must + // apply physical projection per file while returning one consistent partial Variant. + order_qt_jni_mixed_us_projection """ + select id, + cast(payload['name'] as string), + cast(payload['layout'] as string) + from variant_mixed_us + order by id + """ + sql """set force_jni_scanner = false""" explain { @@ -106,6 +168,17 @@ suite("test_paimon_catalog_variant", "p0,external,doris,external_docker,external } } + explain { + sql "select id, cast(payload['name'] as string) from variant_shredded order by id" + check { explainString -> + def nativeSplits = explainString =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + return explainString.contains("all access paths: [payload.name]") + && nativeSplits.find() + && nativeSplits.group(1).toInteger() > 0 + && nativeSplits.group(1) == nativeSplits.group(2) + } + } + order_qt_native_full_variant """ select id, payload from variant_smoke