diff --git a/.licenserc.yaml b/.licenserc.yaml index 67cc48afe085ae..bd648f58d49fd1 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -143,6 +143,10 @@ header: - "docker/thirdparties/docker-compose/kerberos/paimon_data/**" - "docker/thirdparties/docker-compose/kerberos/sql/**" - "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl" + # Generated Paimon schema and snapshot metadata. These files are copied verbatim into + # MinIO by the Iceberg third-party initializer and cannot carry source comments. + - "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/**/schema/**" + - "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/**/snapshot/**" - "docker/thirdparties/docker-compose/postgresql/certs/**" - "docker/thirdparties/docker-compose/mysql/certs/**" - "conf/mysql_ssl_default_certificate/*" diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index f1280435082f38..07b0fef27e9a11 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -61,6 +61,16 @@ namespace doris::format { namespace { +bool requires_paimon_datetime_precision_conversion(const DataTypePtr& source_type, + const DataTypePtr& target_type) { + const auto source = remove_nullable(source_type); + const auto target = remove_nullable(target_type); + const auto primitive = source->get_primitive_type(); + return primitive == target->get_primitive_type() && + (primitive == TYPE_DATETIMEV2 || primitive == TYPE_TIMESTAMPTZ) && + source->get_scale() > target->get_scale(); +} + bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { const auto& table_children = table.identity_children.empty() ? table.children : table.identity_children; @@ -1530,6 +1540,11 @@ static bool mapping_can_use_file_column_directly(const ColumnMapping& mapping) { if (mapping.table_type == nullptr || mapping.file_type == nullptr) { return false; } + // A historical .600000 timestamp narrowed to p0 must be materialized through the + // truncating cast; reusing its file column directly would retain the fraction. + if (mapping.truncate_datetimev2_precision) { + return false; + } const auto table_type = remove_nullable(mapping.table_type); const auto file_type = remove_nullable(mapping.file_type); const bool same_timestamptz_with_different_scale = @@ -1578,6 +1593,11 @@ static FilterConversionType direct_filter_conversion(const ColumnMapping& mappin if (type_contains_varbinary(mapping.table_type)) { return FilterConversionType::FINALIZE_ONLY; } + // Filtering the historical .600000 value before p0 truncation can reject a row + // that equals the whole-second literal after repair. Keep that predicate above materialization. + if (mapping.truncate_datetimev2_precision) { + return FilterConversionType::FINALIZE_ONLY; + } const auto table_type = remove_nullable(mapping.table_type); const auto file_type = remove_nullable(mapping.file_type); // TIMESTAMPTZ scale mismatch is intentionally materialized as pass-through: a SQL cast rounds @@ -2218,7 +2238,8 @@ static void rebuild_projection(ColumnMapping* mapping, LocalIndex block_position return; } - auto expr = EvolutionCast::create_shared(mapping->table_type); + auto expr = EvolutionCast::create_shared(mapping->table_type, + mapping->truncate_datetimev2_precision); expr->add_child(VSlotRef::create_shared(cast_set(block_position.value()), cast_set(block_position.value()), -1, mapping->file_type, mapping->file_column_name)); @@ -2933,6 +2954,11 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c mapping->projected_file_children = file_field.children; mapping->timestamp_is_adjusted_to_utc = file_field.timestamp_is_adjusted_to_utc; mapping->file_type = file_field.type; + const auto file_type = remove_nullable(mapping->file_type); + const auto table_type = remove_nullable(mapping->table_type); + mapping->truncate_datetimev2_precision = + _options.truncate_datetimev2_precision_for_paimon && + requires_paimon_datetime_precision_conversion(file_type, table_type); // Access paths are relative to the Variant terminal, so recursive complex mappings must carry // them instead of leaving them only on the top-level table column. mapping->variant_access_paths = table_column.variant_access_paths; diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 6b6e2d839327ca..41c8cc953f7b86 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -156,6 +156,9 @@ struct ColumnMapping { std::vector child_mappings; // True when file value can be used directly as table value without cast or child remap. bool is_trivial = false; + // Paimon TIMESTAMP precision narrowing uses truncation instead of the ordinary SQL CAST + // rounding behavior. This is carried recursively so nested timestamp children follow it too. + bool truncate_datetimev2_precision = false; // How filters referencing this table/global column can be converted below table-reader // finalize. This is metadata for localize_filters() and future constant-filter evaluation. FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; @@ -174,6 +177,9 @@ struct TableColumnMapperOptions { bool enable_row_lineage_virtual_columns = false; bool enable_iceberg_metadata_virtual_columns = false; bool enable_paimon_metadata_virtual_columns = false; + // Paimon schema evolution truncates timestamp precision. This must not reuse the SQL CAST + // behavior, which rounds when reducing DATETIMEV2 scale. + bool truncate_datetimev2_precision_for_paimon = false; std::string debug_string() const; }; diff --git a/be/src/format_v2/expr/evolution_cast.cpp b/be/src/format_v2/expr/evolution_cast.cpp index 70d878474f7c31..f6234071ed09bc 100644 --- a/be/src/format_v2/expr/evolution_cast.cpp +++ b/be/src/format_v2/expr/evolution_cast.cpp @@ -17,11 +17,63 @@ #include "format_v2/expr/evolution_cast.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_timestamptz.h" #include "exprs/vexpr_context.h" namespace doris::format { +Status EvolutionCast::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { + const auto source = remove_nullable(_children[0]->execute_type(block)); + const auto target = remove_nullable(_data_type); + const auto primitive = source->get_primitive_type(); + // Paimon evolution truncates 01.600000 at p0 to 01; ordinary CAST rounds it to 02. + // Keep this conversion out of SQL casts and other table formats' evolution paths. + if (!_truncate_datetimev2_precision || primitive != target->get_primitive_type() || + (primitive != TYPE_DATETIMEV2 && primitive != TYPE_TIMESTAMPTZ) || + source->get_scale() <= target->get_scale()) { + return Cast::execute_column_impl(context, block, selector, count, result_column); + } + + ColumnPtr input; + RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, input)); + auto result = _data_type->create_column(); + if (is_column_nullable(*result) && !is_column_nullable(*input)) { + auto& nullable_result = assert_cast(*result); + nullable_result.get_nested_column().insert_range_from(*input, 0, count); + nullable_result.get_null_map_column().insert_many_vals(0, count); + } else { + result->insert_range_from(*input, 0, count); + } + IColumn* nested_column = result.get(); + if (is_column_nullable(*nested_column)) { + nested_column = &assert_cast(*nested_column).get_nested_column(); + } + uint32_t divisor = 1; + for (uint32_t i = target->get_scale(); i < 6; ++i) { + divisor *= 10; + } + if (primitive == TYPE_DATETIMEV2) { + auto& data = assert_cast(*nested_column).get_data(); + for (auto& value : data) { + value.unchecked_set_time_unit(value.microsecond() / divisor * + divisor); + } + } else { + auto& data = assert_cast(*nested_column).get_data(); + for (auto& value : data) { + value.set_microsecond(value.microsecond() / divisor * divisor); + } + } + result_column = std::move(result); + return Status::OK(); +} + Status EvolutionCast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR(Cast::prepare(state, desc, context)); diff --git a/be/src/format_v2/expr/evolution_cast.h b/be/src/format_v2/expr/evolution_cast.h index 25b5c02cc70202..592506a24893ee 100644 --- a/be/src/format_v2/expr/evolution_cast.h +++ b/be/src/format_v2/expr/evolution_cast.h @@ -26,15 +26,21 @@ class EvolutionCast final : public Cast { ENABLE_FACTORY_CREATOR(EvolutionCast); public: - explicit EvolutionCast(const DataTypePtr& type) : Cast(type) {} + explicit EvolutionCast(const DataTypePtr& type, bool truncate_datetimev2_precision = false) + : Cast(type), _truncate_datetimev2_precision(truncate_datetimev2_precision) {} Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override; std::string debug_string() const override; Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); - *cloned_expr = EvolutionCast::create_shared(_data_type); + *cloned_expr = EvolutionCast::create_shared(_data_type, _truncate_datetimev2_precision); return Status::OK(); } + +private: + bool _truncate_datetimev2_precision = false; }; } // namespace doris::format diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 76378f186b3238..65d0a8d1d50f24 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -55,6 +55,7 @@ class PaimonReader final : public format::TableReader { protected: void configure_mapper_options(format::TableColumnMapperOptions* options) const override { options->enable_paimon_metadata_virtual_columns = true; + options->truncate_datetimev2_precision_for_paimon = true; } format::TableColumnMappingMode mapping_mode() const override; Status annotate_file_schema(std::vector* file_schema) override; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 928254b177e6f1..9fe42728fa7b4a 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1333,8 +1333,8 @@ class TableReader { } Status _cast_column_to_type(ColumnPtr* column, const DataTypePtr& file_type, - const DataTypePtr& table_type, - const std::string& column_name) const { + const DataTypePtr& table_type, const std::string& column_name, + bool truncate_datetimev2_precision = false) const { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); DORIS_CHECK(file_type != nullptr); @@ -1362,7 +1362,7 @@ class TableReader { Block cast_block; cast_block.insert({*column, input_type, column_name}); auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name); - auto cast_expr = EvolutionCast::create_shared(cast_type); + auto cast_expr = EvolutionCast::create_shared(cast_type, truncate_datetimev2_precision); cast_expr->add_child(std::move(slot_ref)); auto cast_ctx = VExprContext::create_shared(std::move(cast_expr)); RowDescriptor row_desc; @@ -1416,7 +1416,8 @@ class TableReader { // _align_column_nullability() reject an actual NULL before removing the wrapper. ColumnPtr result_column = source.column; RETURN_IF_ERROR(_cast_column_to_type(&result_column, slot->data_type(), mapping.table_type, - mapping.file_column_name)); + mapping.file_column_name, + mapping.truncate_datetimev2_precision)); RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type)); *column = _detach_column(std::move(result_column)); *handled = true; @@ -1436,7 +1437,8 @@ class TableReader { nullable_parent_null_map)); } else { RETURN_IF_ERROR(_cast_column_to_type(column, mapping.file_type, mapping.table_type, - mapping.file_column_name)); + mapping.file_column_name, + mapping.truncate_datetimev2_precision)); } } RETURN_IF_ERROR( diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp index c32d552788de97..902a55bf1851be 100644 --- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp @@ -957,9 +957,8 @@ TEST(DataTypeSerDeArrowTest, PlainArrowConverterDoesNotInferIcebergUuid) { TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) { auto make_variant_column = []() { - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : {std::string_view {R"({"name":"doris","n":1})"}, std::string_view {"null"}, std::string_view {R"([1,true,"x"])"}}) { @@ -1054,9 +1053,8 @@ TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) { TEST(DataTypeSerDeArrowTest, NestedIcebergVariantExtensionsAndParquetSchema) { auto make_variant_column = [](std::initializer_list json_values) { - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : json_values) { encoder.add_json({json.data(), json.size()}); } diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 249c668367398f..82144e55ac115f 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -31,6 +31,7 @@ #include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" @@ -3647,6 +3648,108 @@ TEST_F(ColumnMapperCastTest, ColumnMapperBuildsCastProjectionForTypeMismatch) { mapping.projection->close(); } +TEST_F(ColumnMapperCastTest, PaimonTimestampPrecisionNarrowingTruncatesWithoutRounding) { + const auto file_type = std::make_shared(6); + const auto table_type = std::make_shared(0); + const std::vector table_schema = {name_col("ts", table_type)}; + const std::vector file_schema = {name_col("ts", file_type, 0)}; + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME, + .truncate_datetimev2_precision_for_paimon = true}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].truncate_datetimev2_precision); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok()); + const auto& mapping = mapper.mappings()[0]; + ASSERT_NE(mapping.projection, nullptr); + + auto source = ColumnDateTimeV2::create(); + DateV2Value value; + value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000); + source->get_data().push_back(value); + Block block; + block.insert({std::move(source), file_type, "ts"}); + int result_column_id = -1; + ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok()); + + const auto& result = + assert_cast(*block.get_by_position(result_column_id).column); + ASSERT_EQ(result.size(), 1); + EXPECT_EQ(result.get_data()[0].second(), 1); + EXPECT_EQ(result.get_data()[0].microsecond(), 0); + mapping.projection->close(); +} + +TEST_F(ColumnMapperCastTest, PaimonTimestampTzPrecisionNarrowingTruncatesWithoutRounding) { + const auto file_type = timestamptz(6); + const auto table_type = timestamptz(0); + const std::vector table_schema = {name_col("ts", table_type)}; + const std::vector file_schema = {name_col("ts", file_type, 0)}; + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME, + .truncate_datetimev2_precision_for_paimon = true}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_TRUE(mapper.mappings()[0].truncate_datetimev2_precision); + EXPECT_FALSE(mapper.mappings()[0].is_trivial); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY); + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok()); + const auto& mapping = mapper.mappings()[0]; + ASSERT_NE(mapping.projection, nullptr); + + auto source = ColumnTimeStampTz::create(); + TimestampTzValue value; + value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000); + source->get_data().push_back(value); + Block block; + block.insert({std::move(source), file_type, "ts"}); + int result_column_id = -1; + ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok()); + + const auto& result = + assert_cast(*block.get_by_position(result_column_id).column); + ASSERT_EQ(result.size(), 1); + EXPECT_EQ(result.get_data()[0].second(), 1); + EXPECT_EQ(result.get_data()[0].microsecond(), 0); + mapping.projection->close(); +} + +TEST_F(ColumnMapperCastTest, PaimonTimestampPrecisionNarrowingWrapsRequiredFileColumn) { + const auto file_type = std::make_shared(6); + const auto table_type = make_nullable(std::make_shared(0)); + const std::vector table_schema = {name_col("ts", table_type)}; + const std::vector file_schema = {name_col("ts", file_type, 0)}; + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME, + .truncate_datetimev2_precision_for_paimon = true}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok()); + const auto& mapping = mapper.mappings()[0]; + ASSERT_NE(mapping.projection, nullptr); + + auto source = ColumnDateTimeV2::create(); + DateV2Value value; + value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000); + source->get_data().push_back(value); + Block block; + block.insert({std::move(source), file_type, "ts"}); + int result_column_id = -1; + ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok()); + + const auto& result = + assert_cast(*block.get_by_position(result_column_id).column); + ASSERT_EQ(result.size(), 1); + EXPECT_FALSE(result.is_null_at(0)); + const auto& nested = assert_cast(result.get_nested_column()); + EXPECT_EQ(nested.get_data()[0].second(), 1); + EXPECT_EQ(nested.get_data()[0].microsecond(), 0); + mapping.projection->close(); +} + // Scenario: equivalent table/file types keep the mapping trivial and avoid unnecessary projection casts. TEST_F(ColumnMapperCastTest, ColumnMapperTreatsEquivalentTypesAsTrivial) { TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); diff --git a/be/test/format_v2/expr/evolution_cast_test.cpp b/be/test/format_v2/expr/evolution_cast_test.cpp index e9cd32d656298f..64c48c00484651 100644 --- a/be/test/format_v2/expr/evolution_cast_test.cpp +++ b/be/test/format_v2/expr/evolution_cast_test.cpp @@ -179,4 +179,54 @@ TEST_F(EvolutionCastTest, NumericPromotionUsesExistingConversion) { EXPECT_EQ(column.get_data()[2], 3); } +TEST_F(EvolutionCastTest, PaimonPrecisionNarrowingSurvivesCloneAndPreservesNull) { + for (uint32_t scale : {0, 4, 5}) { + for (bool clone : {false, true}) { + const auto input_type = make_nullable(std::make_shared(6)); + const auto output_type = make_nullable(std::make_shared(scale)); + VExprSPtr expr = EvolutionCast::create_shared(output_type, true); + expr->add_child(VSlotRef::create_shared(0, 0, -1, input_type, "source")); + if (clone) { + VExprSPtr cloned; + ASSERT_TRUE(expr->deep_clone(&cloned).ok()); + expr = std::move(cloned); + } + DateV2Value value; + value.unchecked_set_time(1969, 12, 31, 23, 59, 59, 699999); + Block block; + block.insert({ColumnHelper::create_nullable_column({value, value}, + {0, 1}), + input_type, "source"}); + int result = -1; + ASSERT_TRUE(execute(expr, &block, &result).ok()); + const auto& column = + assert_cast(*block.get_by_position(result).column); + EXPECT_TRUE(column.is_null_at(1)); + const auto& values = assert_cast(column.get_nested_column()); + EXPECT_EQ(values.get_data()[0].second(), 59); + EXPECT_EQ(values.get_data()[0].microsecond(), + scale == 0 ? 0 : (scale == 4 ? 699900 : 699990)); + } + } +} + +TEST_F(EvolutionCastTest, OrdinaryEvolutionDoesNotEnablePaimonTruncation) { + const auto input_type = std::make_shared(6); + const auto output_type = make_nullable(std::make_shared(0)); + DateV2Value value; + value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000); + Block block; + block.insert({ColumnHelper::create_column({value}), input_type, "source"}); + auto expr = EvolutionCast::create_shared(output_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, input_type, "source")); + int result = -1; + const auto status = execute(expr, &block, &result); + ASSERT_TRUE(status.ok()) << status; + const auto& nullable_column = + assert_cast(*block.get_by_position(result).column); + EXPECT_FALSE(nullable_column.is_null_at(0)); + const auto& column = assert_cast(nullable_column.get_nested_column()); + EXPECT_EQ(column.get_data()[0].second(), 2); +} + } // namespace doris::format diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl index 774c83a5db380a..950dba455623bb 100644 --- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl +++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl @@ -207,6 +207,7 @@ services: /usr/bin/mc cp -r /mnt/data/input/minio/warehouse/* minio/warehouse/; fi; /usr/bin/mc cp -r /mnt/preinstalled_data/iceberg/ minio/warehouse/wh/multi_catalog/; + /usr/bin/mc cp -r /mnt/preinstalled_data/paimon/ minio/warehouse/wh/; " networks: diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/bucket-0/data-0e4e0ae4-13ce-495b-a7a0-78a4cdcfe30d-0.orc b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/bucket-0/data-0e4e0ae4-13ce-495b-a7a0-78a4cdcfe30d-0.orc new file mode 100644 index 00000000000000..54c8aacfe571c1 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/bucket-0/data-0e4e0ae4-13ce-495b-a7a0-78a4cdcfe30d-0.orc differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-575ae3ae-ac95-41b9-ac9c-627b9e379898-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-575ae3ae-ac95-41b9-ac9c-627b9e379898-0 new file mode 100644 index 00000000000000..1a013acbef72e8 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-575ae3ae-ac95-41b9-ac9c-627b9e379898-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0 new file mode 100644 index 00000000000000..d6558ffb41073d Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1 new file mode 100644 index 00000000000000..f8b7d52dfaadcf Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-0 new file mode 100644 index 00000000000000..605cae870032ee --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-0 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(6)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc" + }, + "comment" : "", + "timeMillis" : 1790147438974 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-1 new file mode 100644 index 00000000000000..926cb2fc362395 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-1 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(4)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc" + }, + "comment" : "", + "timeMillis" : 1790147459797 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/snapshot-1 new file mode 100644 index 00000000000000..efcd8871ab8d76 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1", + "deltaManifestListSize" : 989, + "changelogManifestList" : null, + "commitUser" : "0a9eb592-62fc-4c8f-a8fb-8effcee26a33", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1790147449999, + "logOffsets" : { }, + "totalRecordCount" : 1, + "deltaRecordCount" : 1, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/bucket-0/data-40c7a472-ba63-4b7d-aa0f-b512cda13f0e-0.parquet b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/bucket-0/data-40c7a472-ba63-4b7d-aa0f-b512cda13f0e-0.parquet new file mode 100644 index 00000000000000..a4d8625cf5301f Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/bucket-0/data-40c7a472-ba63-4b7d-aa0f-b512cda13f0e-0.parquet differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-b482e455-6982-49c7-ac3c-bc618bee7bd5-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-b482e455-6982-49c7-ac3c-bc618bee7bd5-0 new file mode 100644 index 00000000000000..4740cdebe9491f Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-b482e455-6982-49c7-ac3c-bc618bee7bd5-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0 new file mode 100644 index 00000000000000..96c09e2d70267f Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1 new file mode 100644 index 00000000000000..0ad8d570f34f60 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-0 new file mode 100644 index 00000000000000..82033de49ad56c --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-0 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(6)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147403116 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-1 new file mode 100644 index 00000000000000..14696dbbc83c6e --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-1 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(4)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147424631 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/snapshot-1 new file mode 100644 index 00000000000000..484e2f550f679f --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1", + "deltaManifestListSize" : 989, + "changelogManifestList" : null, + "commitUser" : "2fbb418a-d458-4811-a37c-7544fd227009", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1790147415250, + "logOffsets" : { }, + "totalRecordCount" : 1, + "deltaRecordCount" : 1, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/bucket-0/data-aceafc60-6a85-4e6c-9303-3c4523025136-0.parquet b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/bucket-0/data-aceafc60-6a85-4e6c-9303-3c4523025136-0.parquet new file mode 100644 index 00000000000000..41c3e664316de8 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/bucket-0/data-aceafc60-6a85-4e6c-9303-3c4523025136-0.parquet differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-43556a23-7d15-4a7d-b1fe-ef0f6b04615a-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-43556a23-7d15-4a7d-b1fe-ef0f6b04615a-0 new file mode 100644 index 00000000000000..85559b58580b41 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-43556a23-7d15-4a7d-b1fe-ef0f6b04615a-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0 new file mode 100644 index 00000000000000..c66481bff819ee Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1 new file mode 100644 index 00000000000000..0090046e1b321c Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-0 new file mode 100644 index 00000000000000..fcbf662a160277 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-0 @@ -0,0 +1,30 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "trigger_ts", + "type" : "TIMESTAMP(6)" + }, { + "id" : 2, + "name" : "cmap", + "type" : { + "type" : "MAP", + "key" : "TIMESTAMP(6)", + "value" : "TIMESTAMP(6)" + } + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147474751 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-1 new file mode 100644 index 00000000000000..88b11383d03f8d --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-1 @@ -0,0 +1,30 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "trigger_ts", + "type" : "TIMESTAMP(4)" + }, { + "id" : 2, + "name" : "cmap", + "type" : { + "type" : "MAP", + "key" : "TIMESTAMP(6)", + "value" : "TIMESTAMP(6)" + } + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147494142 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/snapshot-1 new file mode 100644 index 00000000000000..a5eb71db5b1e0c --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1", + "deltaManifestListSize" : 983, + "changelogManifestList" : null, + "commitUser" : "8d550109-356f-4c16-b8f7-1007332f9d6b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1790147485248, + "logOffsets" : { }, + "totalRecordCount" : 1, + "deltaRecordCount" : 1, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/bucket-0/data-df5aad5a-827a-4bb9-927d-783200ec15b0-0.parquet b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/bucket-0/data-df5aad5a-827a-4bb9-927d-783200ec15b0-0.parquet new file mode 100644 index 00000000000000..a8a34d91c2f905 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/bucket-0/data-df5aad5a-827a-4bb9-927d-783200ec15b0-0.parquet differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-3940a649-76fc-4fb4-ac46-a4c759baba76-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-3940a649-76fc-4fb4-ac46-a4c759baba76-0 new file mode 100644 index 00000000000000..3a806e6e6c18c7 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-3940a649-76fc-4fb4-ac46-a4c759baba76-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0 new file mode 100644 index 00000000000000..ebc1d84c56d022 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1 new file mode 100644 index 00000000000000..16b30d750dd7f1 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-0 new file mode 100644 index 00000000000000..f90bd1051e9b24 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "nested_array", + "type" : { + "type" : "ARRAY", + "element" : "TIMESTAMP(6)" + } + }, { + "id" : 2, + "name" : "nested_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 3, + "name" : "nested_ts", + "type" : "TIMESTAMP(6)" + } ] + } + } ], + "highestFieldId" : 3, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147507016 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-1 new file mode 100644 index 00000000000000..b7b23c527863e2 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-1 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "nested_array", + "type" : { + "type" : "ARRAY", + "element" : "TIMESTAMP(4)" + } + }, { + "id" : 2, + "name" : "nested_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 3, + "name" : "nested_ts", + "type" : "TIMESTAMP(6)" + } ] + } + } ], + "highestFieldId" : 3, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147527899 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-2 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-2 new file mode 100644 index 00000000000000..3fc1021f16b923 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-2 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 2, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "nested_array", + "type" : { + "type" : "ARRAY", + "element" : "TIMESTAMP(4)" + } + }, { + "id" : 2, + "name" : "nested_row", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 3, + "name" : "nested_ts", + "type" : "TIMESTAMP(4)" + } ] + } + } ], + "highestFieldId" : 3, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1790147535885 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/snapshot-1 new file mode 100644 index 00000000000000..681bc73e1c974d --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1", + "deltaManifestListSize" : 984, + "changelogManifestList" : null, + "commitUser" : "c2b6bc0a-93e5-4917-aa06-c3b9b0e6a57c", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1790147518616, + "logOffsets" : { }, + "totalRecordCount" : 1, + "deltaRecordCount" : 1, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/bucket-0/data-2eb67d55-56cd-44e4-8b28-4884fa0398c1-0.parquet b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/bucket-0/data-2eb67d55-56cd-44e4-8b28-4884fa0398c1-0.parquet new file mode 100644 index 00000000000000..21a9c72eb45cb7 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/bucket-0/data-2eb67d55-56cd-44e4-8b28-4884fa0398c1-0.parquet differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-26ef652b-eb81-4004-88d7-7af675defd83-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-26ef652b-eb81-4004-88d7-7af675defd83-0 new file mode 100644 index 00000000000000..c14ea530dd0127 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-26ef652b-eb81-4004-88d7-7af675defd83-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0 new file mode 100644 index 00000000000000..b0f1a7cb28a459 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1 new file mode 100644 index 00000000000000..faa1fb6a946606 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-0 new file mode 100644 index 00000000000000..1c398accb3a575 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-0 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ntz_to_ltz", + "type" : "TIMESTAMP(6)" + }, { + "id" : 2, + "name" : "ltz_to_ntz", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + }, { + "id" : 3, + "name" : "ntz_to_ltz_control", + "type" : "TIMESTAMP(6)" + }, { + "id" : 4, + "name" : "ltz_to_ntz_control", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "file.format" : "parquet" + }, + "timeMillis" : 1791883714991 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-1 new file mode 100644 index 00000000000000..f0db407c08c53a --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-1 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ntz_to_ltz", + "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE" + }, { + "id" : 2, + "name" : "ltz_to_ntz", + "type" : "TIMESTAMP(0)" + }, { + "id" : 3, + "name" : "ntz_to_ltz_control", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + }, { + "id" : 4, + "name" : "ltz_to_ntz_control", + "type" : "TIMESTAMP(6)" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "file.format" : "parquet" + }, + "timeMillis" : 1791883716179 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/EARLIEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/EARLIEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/snapshot-1 new file mode 100644 index 00000000000000..01c16880770511 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/snapshot-1 @@ -0,0 +1,16 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1", + "deltaManifestListSize" : 1109, + "commitUser" : "9e378f5a-3b40-4a8c-b74d-9bbfb13ff29e", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1791883716073, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/bucket-0/data-e310e0fb-3049-4d92-a9f5-4d021d32327c-0.orc b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/bucket-0/data-e310e0fb-3049-4d92-a9f5-4d021d32327c-0.orc new file mode 100644 index 00000000000000..cd004e3a5cdce7 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/bucket-0/data-e310e0fb-3049-4d92-a9f5-4d021d32327c-0.orc differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-2223cd32-36ff-4f7e-92ea-b60bfa02f487-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-2223cd32-36ff-4f7e-92ea-b60bfa02f487-0 new file mode 100644 index 00000000000000..8c67f41ee7d7ee Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-2223cd32-36ff-4f7e-92ea-b60bfa02f487-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-0 new file mode 100644 index 00000000000000..8874b602d09224 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-1 new file mode 100644 index 00000000000000..94a208b2f04407 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-0 new file mode 100644 index 00000000000000..814a84e6c32b8a --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-0 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(9)" + }, { + "id" : 2, + "name" : "ltz", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc", + "orc.timestamp-ltz.legacy.type" : "false" + }, + "comment" : "", + "timeMillis" : 1791801815350 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-1 new file mode 100644 index 00000000000000..e66724d3708cac --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-1 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(0)" + }, { + "id" : 2, + "name" : "ltz", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc", + "orc.timestamp-ltz.legacy.type" : "false" + }, + "comment" : "", + "timeMillis" : 1791801821110 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-2 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-2 new file mode 100644 index 00000000000000..a548c88ab82af4 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-2 @@ -0,0 +1,27 @@ +{ + "version" : 3, + "id" : 2, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(0)" + }, { + "id" : 2, + "name" : "ltz", + "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc", + "orc.timestamp-ltz.legacy.type" : "false" + }, + "comment" : "", + "timeMillis" : 1791801821222 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/snapshot-1 new file mode 100644 index 00000000000000..a31bcb7fe26329 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-1", + "deltaManifestListSize" : 984, + "changelogManifestList" : null, + "commitUser" : "47e8140c-7f94-4b15-b046-e2b09c0a319a", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1791801819770, + "logOffsets" : { }, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/bucket-0/data-0449dfa1-48bb-41be-813f-4195bf9bee95-0.orc b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/bucket-0/data-0449dfa1-48bb-41be-813f-4195bf9bee95-0.orc new file mode 100644 index 00000000000000..049f6fb10af332 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/bucket-0/data-0449dfa1-48bb-41be-813f-4195bf9bee95-0.orc differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-1ec1b6cb-4403-41d6-81a6-9b7fd3f5273f-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-1ec1b6cb-4403-41d6-81a6-9b7fd3f5273f-0 new file mode 100644 index 00000000000000..95f00536be94a3 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-1ec1b6cb-4403-41d6-81a6-9b7fd3f5273f-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0 new file mode 100644 index 00000000000000..ffb80803777957 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1 new file mode 100644 index 00000000000000..d4b54297598d90 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-0 new file mode 100644 index 00000000000000..29efda5f810b97 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-0 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(6)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc" + }, + "comment" : "", + "timeMillis" : 1791723191367 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-1 new file mode 100644 index 00000000000000..d1c1c1d3cc3daf --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-1 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(0)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "orc" + }, + "comment" : "", + "timeMillis" : 1791723302472 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/snapshot-1 new file mode 100644 index 00000000000000..9aeb6d472cbac8 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1", + "deltaManifestListSize" : 989, + "changelogManifestList" : null, + "commitUser" : "8ad87a12-6094-4711-a5d1-2af226208a48", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1791723194154, + "logOffsets" : { }, + "totalRecordCount" : 3, + "deltaRecordCount" : 3, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/bucket-0/data-501848c5-b1cf-4c3e-af8f-103b7d377827-0.parquet b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/bucket-0/data-501848c5-b1cf-4c3e-af8f-103b7d377827-0.parquet new file mode 100644 index 00000000000000..ba0b1241963bf0 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/bucket-0/data-501848c5-b1cf-4c3e-af8f-103b7d377827-0.parquet differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-05e65016-fbe3-40e3-a0d9-2dcc01cb4c2b-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-05e65016-fbe3-40e3-a0d9-2dcc01cb4c2b-0 new file mode 100644 index 00000000000000..777df9be3f5d5c Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-05e65016-fbe3-40e3-a0d9-2dcc01cb4c2b-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-0 new file mode 100644 index 00000000000000..069985cc8ac532 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-0 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-1 new file mode 100644 index 00000000000000..0547e56d7cb925 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-1 differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-0 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-0 new file mode 100644 index 00000000000000..6cc37d3a904b40 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-0 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(6)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1791722933833 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-1 new file mode 100644 index 00000000000000..777472cad7f0a7 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-1 @@ -0,0 +1,22 @@ +{ + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts", + "type" : "TIMESTAMP(0)" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "write-only" : "true", + "file.format" : "parquet" + }, + "comment" : "", + "timeMillis" : 1791723086473 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/LATEST b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/LATEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/snapshot-1 new file mode 100644 index 00000000000000..8084e00330f07c --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-1", + "deltaManifestListSize" : 983, + "changelogManifestList" : null, + "commitUser" : "5871947f-0fd6-4241-93a9-e583190bdd86", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1791722936925, + "logOffsets" : { }, + "totalRecordCount" : 3, + "deltaRecordCount" : 3, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index c8625802e1f12d..3ca105bf317371 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -32,6 +32,7 @@ import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,7 +185,8 @@ public LocalDate getDate() { @Override public LocalDateTime getDateTime() { - Timestamp ts = record.getTimestamp(idx, dorisType.getPrecision()); + Timestamp ts = truncateTimestampPrecision( + record.getTimestamp(idx, getPaimonTimestampPrecision())); if (dataType instanceof LocalZonedTimestampType) { // Paimon stores TIMESTAMP_LTZ as an epoch instant, so convert it directly in the cached session zone. return LocalDateTime.ofInstant(ts.toInstant(), timeZone); @@ -195,11 +197,45 @@ public LocalDateTime getDateTime() { @Override public LocalDateTime getTimeStampTz() { - Timestamp ts = record.getTimestamp(idx, dorisType.getPrecision()); + Timestamp ts = truncateTimestampPrecision( + record.getTimestamp(idx, getPaimonTimestampPrecision())); // Timestamp's local representation is identical to converting its epoch instant in UTC. return ts.toLocalDateTime(); } + Timestamp truncateTimestampPrecision(Timestamp timestamp) { + if (!(dataType instanceof TimestampType) && !(dataType instanceof LocalZonedTimestampType)) { + return timestamp; + } + int precision = dorisType.getPrecision(); + return truncateTimestampPrecision(timestamp, precision); + } + + static Timestamp truncateTimestampPrecision(Timestamp timestamp, int precision) { + long divisor = 1; + for (int i = precision; i < 9; ++i) { + divisor *= 10; + } + long nano = Math.floorMod(timestamp.getMillisecond(), 1000L) * 1_000_000L + + timestamp.getNanoOfMillisecond(); + if (nano % divisor == 0) { + return timestamp; + } + LocalDateTime value = timestamp.toLocalDateTime(); + int truncatedNano = (int) (nano / divisor * divisor); + return Timestamp.fromLocalDateTime(value.withNano(truncatedNano)); + } + + private int getPaimonTimestampPrecision() { + if (dataType instanceof TimestampType) { + return ((TimestampType) dataType).getPrecision(); + } + if (dataType instanceof LocalZonedTimestampType) { + return ((LocalZonedTimestampType) dataType).getPrecision(); + } + return dorisType.getPrecision(); + } + @Override public boolean isNull() { boolean isNull = record.isNullAt(idx); diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index c7b410c3102231..b6b29134070fb0 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -30,19 +30,32 @@ import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.options.MemorySize; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.reader.FileRecordIterator; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.ChainGroupReadTable; import org.apache.paimon.table.DelegatedFileStoreTable; import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FallbackReadFileStoreTable.FallbackSplit; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.QueryAuthSplit; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.system.AuditLogTable; +import org.apache.paimon.table.system.BinlogTable; +import org.apache.paimon.table.system.ReadOptimizedTable; +import org.apache.paimon.table.system.RowTrackingTable; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VariantType; @@ -91,6 +104,7 @@ public class PaimonJniScanner extends JniScanner { "doris.scan.manifest.parallelism-cap"; static final String DORIS_SERIALIZED_SYSTEM_SOURCE = "doris.serialized-system-source"; static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type"; + static final String DORIS_TIMESTAMP_PRECISION_REPAIR = "doris.timestamp-precision-repair"; private static final String SERIALIZED_SYSTEM_SOURCE = PAIMON_OPTION_PREFIX + DORIS_SERIALIZED_SYSTEM_SOURCE; static final String ENABLE_JNI_IO_MANAGER = "paimon.jni.enable_jni_io_manager"; @@ -124,6 +138,7 @@ public class PaimonJniScanner extends JniScanner { private final ClassLoader classLoader; private PreExecutionAuthenticator preExecutionAuthenticator; private boolean scannerCounted; + private boolean requiresDatetimeV2PrecisionRepair; private long openTimeNanos; private long readBatchTimeNanos; private long readBatchCalls; @@ -182,8 +197,6 @@ protected void openInternal() throws IOException { initTableAndReader(); return null; }); - resetDatetimeV2Precision(); - } catch (Throwable e) { try { close(); @@ -198,6 +211,18 @@ protected void openInternal() throws IOException { } private void initReader() throws IOException { + Split split = getSplit(); + QueryAuthSplit authSplit = PaimonTimestampQueryAuth.findAuthSplit(split); + Table dataSource = dataSource(table); + RowType policyTableType = policyRowType(dataSource, split); + if (authSplit != null && authSplit.authResult() != null + && Boolean.parseBoolean(params.getOrDefault( + PAIMON_OPTION_PREFIX + DORIS_TIMESTAMP_PRECISION_REPAIR, "true")) + && PaimonTimestampQueryAuth.readsTimestamp(authSplit.authResult(), policyTableType, + Arrays.stream(fields).filter(field -> !isFileMetadataField(field)) + .collect(Collectors.toList()))) { + requiresDatetimeV2PrecisionRepair = true; + } ReadBuilder readBuilder = table.newReadBuilder(); if (Arrays.stream(fields).filter(field -> !isFileMetadataField(field)).count() > paimonAllFieldNames.size()) { throw new IOException( @@ -208,6 +233,7 @@ private void initReader() throws IOException { } int[] projected = getProjected(); List readFields = new ArrayList<>(projected.length); + List currentFields = new ArrayList<>(projected.length); variantProjections = new ArrayList<>(projected.length); boolean hasReadTypeProjection = false; outputToReadIndex = new int[fields.length]; @@ -219,17 +245,33 @@ private void initReader() throws IOException { } outputToReadIndex[outputIndex] = readIndex; DataField tableField = table.rowType().getFields().get(projected[readIndex]); + if (requiresDatetimeV2PrecisionRepair && dataSource instanceof FallbackReadFileStoreTable + && !SpecialFields.isSystemField(tableField.name())) { + // Fallback schemas agree positionally, but their names and field IDs are independent. + // Resolve positions in the data schema, not a system wrapper's rowkind-prefixed schema. + int dataIndex = dataSource.rowType().getFieldIndex(tableField.name()); + Preconditions.checkState(dataIndex >= 0, "Data field %s not found in schema", tableField.name()); + tableField = policyTableType.getFields().get(dataIndex); + if (table instanceof BinlogTable) { + // Binlog exposes each data field as an array, even when its branch field is scalar. + // Preserve that wrapper while binding the branch ID, as BinlogTable.rowType() does. + tableField = tableField.newType(new ArrayType(tableField.type().nullable())); + } + } PaimonVariantProjection variantProjection = tableField.type() instanceof VariantType ? PaimonVariantProjection.create( variantAccessPathsByColumn.get(outputIndex), timeZone) : null; variantProjections.add(variantProjection); if (variantProjection == null) { - // The engine may have pruned this complex column down to the sub-fields the query touches. - // Mirroring that shape with paimon's own types lets withReadType push the same projection - // through the ROW/ARRAY/MAP readers instead of reading the whole column and discarding it. DataType projectedType = PaimonReadTypeProjection.project(tableField.type(), types[outputIndex]); + // ColumnType retains the logical precision, independently of the widened physical read type. + resetTimestampPrecision(types[outputIndex], projectedType); + currentFields.add(tableField.newType(projectedType)); + if (requiresDatetimeV2PrecisionRepair) { + projectedType = createSafeTimestampReadType(projectedType); + } if (projectedType != tableField.type()) { hasReadTypeProjection = true; } @@ -238,19 +280,102 @@ private void initReader() throws IOException { hasReadTypeProjection = true; readFields.add(tableField.newType( variantProjection.readType().copy(tableField.type().isNullable()))); + currentFields.add(readFields.get(readFields.size() - 1)); } outputIndex++; } + boolean repairPolicy = requiresDatetimeV2PrecisionRepair && authSplit != null + && authSplit.authResult() != null; + List outputReadFields = new ArrayList<>(readFields); + RowType outputType = new RowType(currentFields); + if (repairPolicy) { + PaimonTimestampQueryAuth.addPolicyFields( + authSplit.authResult(), policyTableType, readFields, currentFields); + split = PaimonTimestampQueryAuth.repairPolicy(split, new RowType(currentFields)); + hasReadTypeProjection = true; + } if (hasReadTypeProjection) { readBuilder.withReadType(new RowType(readFields)); } else { readBuilder.withProjection(projected); } + // FE removes only predicates that depend on timestamp values being repaired. Safe + // non-timestamp predicates remain useful for Paimon file and row pruning. readBuilder.withFilter(getPredicates()); - reader = newReadWithOptionalIOManager(readBuilder).executeFilter().createReader(getSplit()); + reader = newReadWithOptionalIOManager(readBuilder).executeFilter().createReader(split); + if (repairPolicy) { + reader = PaimonTimestampQueryAuth.projectOutput(reader, new RowType(currentFields), outputType); + } // Decode with the types that were REQUESTED, not the table's: a pruned nested column comes back // narrow, and taking table.rowType() here would misalign every one of them. - paimonDataTypeList = readFields.stream().map(DataField::type).collect(Collectors.toList()); + paimonDataTypeList = outputReadFields.stream().map(DataField::type).collect(Collectors.toList()); + } + + private static Table dataSource(Table source) { + if (source instanceof FileStoreTable) { + return unwrapSystemPlanningSource((FileStoreTable) source); + } + if (!(source instanceof AuditLogTable || source instanceof ReadOptimizedTable + || source instanceof RowTrackingTable)) { + return source; + } + // A row-backed system wrapper synthesizes its system fields outside the data reader. + // Resolve the pinned data source, without reopening the catalog. + Optional systemSource = hiddenSystemSource(source); + return systemSource.isPresent() ? dataSource(systemSource.get()) : source; + } + + private static RowType policyRowType(Table source, Split split) { + source = dataSource(source); + if (source instanceof ChainGroupReadTable) { + // ChainSplit has no fallback flag: the SDK always reads it through the delta table. + // Using the snapshot schema here can turn a stored delta ts#2 into a missing ts#3. + return policyRowType(((ChainGroupReadTable) source).other(), split); + } + if (source instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) source; + boolean readFallback = split instanceof FallbackSplit && ((FallbackSplit) split).isFallback(); + FileStoreTable branch = isWrappedFirst(pair) != readFallback ? pair.wrapped() : pair.other(); + // Mirror the SDK reader's branch dispatch. Hidden policy fields belong to that + // branch's schema, not the outer table's independently named columns. + return policyRowType(branch, readFallback ? ((FallbackSplit) split).wrapped() : split); + } + return source.rowType(); + } + + static RowType createSafeTimestampReadType(RowType tableType) { + List fields = tableType.getFields().stream() + .map(field -> field.newType(createSafeTimestampReadType(field.type()))) + .collect(Collectors.toList()); + return tableType.copy(fields); + } + + static DataType createSafeTimestampReadType(DataType dataType) { + if (dataType instanceof TimestampType) { + return new TimestampType(dataType.isNullable(), TimestampType.MAX_PRECISION); + } + if (dataType instanceof LocalZonedTimestampType) { + return new LocalZonedTimestampType(dataType.isNullable(), LocalZonedTimestampType.MAX_PRECISION); + } + if (dataType instanceof RowType) { + RowType rowType = (RowType) dataType; + List fields = rowType.getFields().stream() + .map(field -> field.newType(createSafeTimestampReadType(field.type()))) + .collect(Collectors.toList()); + return rowType.copy(fields); + } + if (dataType instanceof ArrayType) { + ArrayType arrayType = (ArrayType) dataType; + return arrayType.newElementType(createSafeTimestampReadType(arrayType.getElementType())); + } + if (dataType instanceof MapType) { + MapType mapType = (MapType) dataType; + // Paimon 1.3.1 cannot cast MAP keys during schema evolution. Timestamp keys are + // already part of the map identity and widening them makes reader construction fail. + return mapType.newKeyValueType(mapType.getKeyType(), + createSafeTimestampReadType(mapType.getValueType())); + } + return dataType; } private TableRead newReadWithOptionalIOManager(ReadBuilder readBuilder) throws IOException { @@ -367,6 +492,18 @@ private List getPredicates() { return Collections.emptyList(); } List predicates = PaimonUtils.deserialize(paimonPredicate); + if (requiresDatetimeV2PrecisionRepair + && !params.containsKey(PAIMON_OPTION_PREFIX + DORIS_TIMESTAMP_PRECISION_REPAIR)) { + // Legacy FE did not classify its predicate list. Preserve independent AND siblings, + // but keep timestamp-dependent subtrees (including mixed OR) for the Doris residual. + java.util.Set timestamps = table.rowType().getFields().stream() + .filter(field -> containsTimestampType(field.type())) + .map(DataField::name).collect(Collectors.toSet()); + predicates = predicates.stream().flatMap(predicate -> PredicateBuilder.splitAnd(predicate).stream()) + .filter(predicate -> PredicateVisitor.collectFieldNames(predicate).stream() + .noneMatch(timestamps::contains)) + .collect(Collectors.toList()); + } if (LOG.isDebugEnabled()) { LOG.debug("predicates:{}", predicates); } @@ -381,22 +518,59 @@ private Split getSplit() { return split; } - private void resetDatetimeV2Precision() { - for (int i = 0; i < types.length; i++) { - if (types[i].isDateTimeV2()) { - // paimon support precision > 6, but it has been reset as 6 in FE - // try to get the right precision for datetimev2 - int index = getFieldIndex(paimonAllFieldNames, fields[i]); - if (index != -1) { - DataType dataType = table.rowType().getTypeAt(index); - if (dataType instanceof TimestampType) { - types[i].setPrecision(((TimestampType) dataType).getPrecision()); - } - } + private void initTimestampPrecisionRepair() { + boolean repairHistoricalPrecision = Boolean.parseBoolean( + params.getOrDefault(PAIMON_OPTION_PREFIX + DORIS_TIMESTAMP_PRECISION_REPAIR, "true")); + for (ColumnType type : types) { + if (repairHistoricalPrecision && containsTimestampType(type)) { + requiresDatetimeV2PrecisionRepair = true; } } } + static void resetTimestampPrecision(ColumnType outputType, DataType currentType) { + if (currentType instanceof TimestampType || currentType instanceof LocalZonedTimestampType) { + outputType.setPrecision(DataTypeChecks.getPrecision(currentType)); + } else if (currentType instanceof ArrayType) { + resetTimestampPrecision(outputType.getChildTypes().get(0), ((ArrayType) currentType).getElementType()); + } else if (currentType instanceof MapType) { + MapType map = (MapType) currentType; + resetTimestampPrecision(outputType.getChildTypes().get(0), map.getKeyType()); + resetTimestampPrecision(outputType.getChildTypes().get(1), map.getValueType()); + } else if (currentType instanceof RowType) { + RowType row = (RowType) currentType; + for (int i = 0; i < row.getFieldCount(); i++) { + resetTimestampPrecision(outputType.getChildTypes().get(i), row.getTypeAt(i)); + } + } + } + + static boolean containsTimestampType(ColumnType type) { + if (type.isDateTimeV2() || type.getType() == ColumnType.Type.TIMESTAMPTZ + || type.getType() == ColumnType.Type.TIMESTAMP_NS) { + return true; + } + List children = type.getChildTypes(); + return children != null && children.stream().anyMatch(PaimonJniScanner::containsTimestampType); + } + + static boolean containsTimestampType(DataType type) { + if (type instanceof TimestampType || type instanceof LocalZonedTimestampType) { + return true; + } + if (type instanceof RowType) { + return ((RowType) type).getFields().stream().anyMatch(field -> containsTimestampType(field.type())); + } + if (type instanceof ArrayType) { + return containsTimestampType(((ArrayType) type).getElementType()); + } + if (type instanceof MapType) { + MapType map = (MapType) type; + return containsTimestampType(map.getKeyType()) || containsTimestampType(map.getValueType()); + } + return false; + } + @Override protected void closeInternal() throws IOException { IOException exception = null; @@ -981,10 +1155,12 @@ private boolean initTableFromCache() { private void initTableAndReader() throws IOException { if (initTableFromCache()) { + initTimestampPrecisionRepair(); initReader(); return; } initTable(); + initTimestampPrecisionRepair(); initReader(); PaimonTableCache.TableCacheEntry candidate = new PaimonTableCache.TableCacheEntry(table, paimonAllFieldNames); diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTimestampQueryAuth.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTimestampQueryAuth.java new file mode 100644 index 00000000000000..f7ca29b424c938 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTimestampQueryAuth.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateVisitor; +import org.apache.paimon.predicate.Transform; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.FallbackReadFileStoreTable.FallbackSplitImpl; +import org.apache.paimon.table.source.QueryAuthSplit; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.utils.InternalRowUtils; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** Keeps SDK authorization and masking after current-precision materialization. */ +final class PaimonTimestampQueryAuth extends TableQueryAuthResult { + private static final long serialVersionUID = 1L; + private final RowType currentReadType; + private final TableQueryAuthResult original; + + PaimonTimestampQueryAuth(TableQueryAuthResult original, RowType currentReadType) { + super(original.filter(), original.columnMasking()); + this.original = original; + this.currentReadType = currentReadType; + } + + @Override + public Predicate extractPredicate() { + return original.extractPredicate(); + } + + @Override + public Map extractColumnMasking() { + return original.extractColumnMasking(); + } + + static QueryAuthSplit findAuthSplit(Split split) { + // FallbackDataSplit is a leaf whose wrapped() returns itself, not a split container. + if (split instanceof FallbackSplitImpl) { + return findAuthSplit(((FallbackSplitImpl) split).wrapped()); + } + return split instanceof QueryAuthSplit ? (QueryAuthSplit) split : null; + } + + static Split repairPolicy(Split split, RowType currentReadType) { + if (split instanceof FallbackSplitImpl) { + FallbackSplitImpl fallback = (FallbackSplitImpl) split; + return new FallbackSplitImpl(repairPolicy(fallback.wrapped(), currentReadType), fallback.isFallback()); + } + QueryAuthSplit auth = (QueryAuthSplit) split; + return new QueryAuthSplit(auth.split(), new PaimonTimestampQueryAuth(auth.authResult(), currentReadType)); + } + + static void addPolicyFields(TableQueryAuthResult auth, RowType tableType, + List readFields, List currentFields) { + List projected = readFields.stream().map(DataField::name).collect(Collectors.toList()); + Set required = requiredFields(auth, new LinkedHashSet<>(projected)); + for (DataField field : tableType.getFields()) { + if (required.contains(field.name())) { + int index = projected.indexOf(field.name()); + DataField widened = field.newType(PaimonJniScanner.createSafeTimestampReadType(field.type())); + if (index < 0) { + currentFields.add(field); + readFields.add(widened); + } else if (field.type() instanceof RowType || field.type() instanceof ArrayType + || field.type() instanceof MapType) { + // A policy can read a sibling that SQL pruned from a projected complex column. + currentFields.set(index, field); + readFields.set(index, widened); + } + } + } + } + + static boolean readsTimestamp(TableQueryAuthResult auth, RowType tableType, List projected) { + Set required = requiredFields(auth, new LinkedHashSet<>(projected)); + return tableType.getFields().stream().anyMatch(field -> required.contains(field.name()) + && PaimonJniScanner.containsTimestampType(field.type())); + } + + private static Set requiredFields(TableQueryAuthResult auth, Set projected) { + Set required = new LinkedHashSet<>(); + Predicate filter = auth.extractPredicate(); + if (filter != null) { + required.addAll(PredicateVisitor.collectFieldNames(filter)); + } + for (Map.Entry mask : auth.extractColumnMasking().entrySet()) { + if (!projected.contains(mask.getKey())) { + continue; + } + required.add(mask.getKey()); + for (Object input : mask.getValue().inputs()) { + if (input instanceof FieldRef) { + required.add(((FieldRef) input).name()); + } + } + } + return required; + } + + static RecordReader projectOutput(RecordReader reader, + RowType currentReadType, RowType outputType) { + if (currentReadType.equals(outputType)) { + return reader; + } + return reader.transform(row -> repairRow(row, currentReadType, outputType)); + } + + @Override + public RecordReader doAuth(RecordReader reader, RowType readType) { + // Delegate the policy itself to Paimon; neither filters nor masks are dropped or rewritten. + // System-table readers synthesize fields such as rowkind only after this inner callback. + RowType currentDataType = currentReadType.project(readType.getFieldNames()); + if (readType.equals(currentDataType)) { + return original.doAuth(reader, readType); + } + return original.doAuth(reader.transform(row -> repairRow(row, readType, currentDataType)), currentDataType); + } + + static InternalRow repairRow(InternalRow row, RowType readType, RowType currentType) { + GenericRow result = new GenericRow(row.getRowKind(), currentType.getFieldCount()); + for (int i = 0; i < currentType.getFieldCount(); i++) { + int sourceIndex = readType.getFieldIndex(currentType.getFields().get(i).name()); + DataType source = readType.getTypeAt(sourceIndex); + result.setField(i, repairValue(InternalRowUtils.get(row, sourceIndex, source), + source, currentType.getTypeAt(i))); + } + return result; + } + + private static Object repairValue(Object value, DataType readType, DataType currentType) { + if (value == null || readType.equals(currentType)) { + return value; + } + if (currentType instanceof TimestampType || currentType instanceof LocalZonedTimestampType) { + return PaimonColumnValue.truncateTimestampPrecision( + (Timestamp) value, DataTypeChecks.getPrecision(currentType)); + } + if (currentType instanceof RowType) { + return repairRow((InternalRow) value, (RowType) readType, (RowType) currentType); + } + if (currentType instanceof ArrayType) { + InternalArray array = (InternalArray) value; + Object[] elements = new Object[array.size()]; + DataType source = ((ArrayType) readType).getElementType(); + DataType target = ((ArrayType) currentType).getElementType(); + for (int i = 0; i < elements.length; i++) { + elements[i] = repairValue(InternalRowUtils.get(array, i, source), source, target); + } + return new GenericArray(elements); + } + if (currentType instanceof MapType) { + InternalMap map = (InternalMap) value; + MapType source = (MapType) readType; + DataType target = ((MapType) currentType).getValueType(); + Map entries = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + entries.put(InternalRowUtils.get(map.keyArray(), i, source.getKeyType()), + repairValue(InternalRowUtils.get(map.valueArray(), i, source.getValueType()), + source.getValueType(), target)); + } + return new GenericMap(entries); + } + return value; + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java index f8c8a34ddb26ce..75950444b7c574 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java @@ -228,6 +228,54 @@ public void testNestedArrayCacheRetainsOnlyCurrentShape() throws Exception { Assertions.assertEquals(outerSize + innerSize, retainedCachedValues(arrayValue)); } + @Test + public void testNanosecondPrecisionInNestedOutputLeaves() { + for (int precision : new int[] {7, 8}) { + TimestampType timestampType = new TimestampType(precision); + RowType current = RowType.of(new DataType[] { + new ArrayType(timestampType), new MapType(new IntType(), timestampType), + RowType.of(new DataType[] {timestampType}, new String[] {"ts"})}, + new String[] {"array", "map", "nested"}); + ColumnType output = ColumnType.parseType("root", "struct," + + "map:map,nested:struct>"); + PaimonJniScanner.resetTimestampPrecision(output, current); + LocalDateTime value = LocalDateTime.of(2025, 1, 1, 0, 0, 1, 123456789); + Timestamp timestamp = Timestamp.fromLocalDateTime(value); + GenericRow row = GenericRow.of(new GenericArray(new Object[] {timestamp}), + new GenericMap(Collections.singletonMap(7, timestamp)), GenericRow.of(timestamp)); + PaimonColumnValue root = new PaimonColumnValue(GenericRow.of(row), 0, output, + PaimonJniScanner.createSafeTimestampReadType(current), "UTC"); + List fields = new ArrayList<>(); + root.unpackStruct(Arrays.asList(0, 1, 2), fields); + List array = new ArrayList<>(); + fields.get(0).unpackArray(array); + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + fields.get(1).unpackMap(keys, values); + List nested = new ArrayList<>(); + fields.get(2).unpackStruct(Collections.singletonList(0), nested); + LocalDateTime expected = value.withNano(precision == 7 ? 123456700 : 123456780); + Assertions.assertEquals(expected, array.get(0).getDateTime()); + Assertions.assertEquals(7, keys.get(0).getInt()); + Assertions.assertEquals(expected, values.get(0).getDateTime()); + Assertions.assertEquals(expected, nested.get(0).getDateTime()); + } + } + + @Test + public void testAlignedTimestampReusesOriginalValue() { + for (int precision : new int[] {0, 4, 6, 9}) { + int nano = precision == 0 ? 0 : 123_400_000; + Timestamp timestamp = Timestamp.fromLocalDateTime( + LocalDateTime.of(1969, 12, 31, 23, 59, 59, nano)); + for (DataType type : new DataType[] {new TimestampType(9), new LocalZonedTimestampType(9)}) { + PaimonColumnValue value = new PaimonColumnValue(GenericRow.of(timestamp), 0, + ColumnType.parseType("t", "datetimev2(" + precision + ")"), type, "UTC"); + Assertions.assertSame(timestamp, value.truncateTimestampPrecision(timestamp)); + } + } + } + @Test public void testTimestampConversionsPreserveBoundaryValues() { Timestamp timestamp = Timestamp.fromEpochMillis(-1, 999_999); @@ -242,6 +290,26 @@ public void testTimestampConversionsPreserveBoundaryValues() { LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999_999_999), timestampValue.getTimeStampTz()); + PaimonColumnValue secondsPrecisionValue = new PaimonColumnValue( + GenericRow.of(Timestamp.fromEpochMillis(1_735_689_601_600L)), 0, + ColumnType.parseType("t", "datetimev2(0)"), new TimestampType(0), "UTC"); + Assertions.assertEquals( + LocalDateTime.of(2025, 1, 1, 0, 0, 1), secondsPrecisionValue.getDateTime()); + + PaimonColumnValue leadingZeroValue = new PaimonColumnValue( + GenericRow.of(Timestamp.fromLocalDateTime( + LocalDateTime.of(2025, 1, 1, 0, 0, 1, 1_000))), 0, + ColumnType.parseType("t", "datetimev2(4)"), new TimestampType(6), "UTC"); + Assertions.assertEquals( + LocalDateTime.of(2025, 1, 1, 0, 0, 1), leadingZeroValue.getDateTime()); + + PaimonColumnValue preEpochValue = new PaimonColumnValue( + GenericRow.of(Timestamp.fromLocalDateTime( + LocalDateTime.of(1969, 12, 31, 23, 59, 59, 600_000_000))), 0, + ColumnType.parseType("t", "datetimev2(0)"), new TimestampType(6), "UTC"); + Assertions.assertEquals( + LocalDateTime.of(1969, 12, 31, 23, 59, 59), preEpochValue.getDateTime()); + PaimonColumnValue localZonedValue = new PaimonColumnValue( GenericRow.of(Timestamp.fromInstant(Instant.parse("2024-03-10T10:30:00.123456789Z"))), 0, dorisTimestampType, new LocalZonedTimestampType(9), "America/Los_Angeles"); @@ -249,6 +317,17 @@ public void testTimestampConversionsPreserveBoundaryValues() { LocalDateTime.of(2024, 3, 10, 3, 30, 0, 123_456_789), localZonedValue.getDateTime()); + PaimonColumnValue narrowedLocalZonedValue = new PaimonColumnValue( + GenericRow.of(Timestamp.fromInstant(Instant.parse("2024-03-10T10:30:00.123456789Z"))), 0, + ColumnType.parseType("t", "datetimev2(0)"), new LocalZonedTimestampType(9), + "America/Los_Angeles"); + Assertions.assertEquals( + LocalDateTime.of(2024, 3, 10, 3, 30), + narrowedLocalZonedValue.getDateTime()); + Assertions.assertEquals( + LocalDateTime.of(2024, 3, 10, 10, 30), + narrowedLocalZonedValue.getTimeStampTz()); + localZonedValue.setTimeZone("Asia/Shanghai"); Assertions.assertEquals( LocalDateTime.of(2024, 3, 10, 18, 30, 0, 123_456_789), diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index b4cabd0f424e3d..63a034d6100590 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -42,9 +42,14 @@ import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.TableRead; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; import org.apache.paimon.utils.InstantiationUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -90,6 +95,157 @@ public void clearTableCache() { PaimonTableCache.clearForTest(); } + @Test + public void crossFamilyHistoricalParquetTimestampsRetainFractionsUntilRepair() throws Exception { + org.apache.doris.jni.spi.utils.OffHeap.setTesting(); + java.util.TimeZone originalZone = java.util.TimeZone.getDefault(); + try (org.apache.paimon.catalog.Catalog catalog = new org.apache.paimon.catalog.FileSystemCatalog( + org.apache.paimon.fs.local.LocalFileIO.create(), + new org.apache.paimon.fs.Path(temporaryFolder.toUri()))) { + catalog.createDatabase("db", false); + for (String zone : new String[] {"UTC", "Asia/Shanghai"}) { + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone(zone)); + for (boolean toLtz : new boolean[] {false, true}) { + DataType historical = toLtz ? DataTypes.TIMESTAMP(6) : new LocalZonedTimestampType(6); + DataType current = toLtz ? new LocalZonedTimestampType(0) : DataTypes.TIMESTAMP(0); + Identifier id = Identifier.create("db", "cross_family_" + toLtz + "_" + zone.replace('/', '_')); + catalog.createTable(id, org.apache.paimon.schema.Schema.newBuilder() + .column("ts", historical).option("file.format", "parquet").build(), false); + Table table = catalog.getTable(id); + java.time.LocalDateTime[] values = { + java.time.LocalDateTime.of(1969, 12, 31, 23, 59, 59, 600_000_000), + java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1, 1000)}; + org.apache.paimon.table.sink.BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (org.apache.paimon.table.sink.BatchTableWrite write = writer.newWrite()) { + for (java.time.LocalDateTime value : values) { + write.write(org.apache.paimon.data.GenericRow.of( + org.apache.paimon.data.Timestamp.fromLocalDateTime(value))); + } + try (org.apache.paimon.table.sink.BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(id, Collections.singletonList( + org.apache.paimon.schema.SchemaChange.updateColumnType("ts", current, true)), false); + table = catalog.getTable(id); + org.apache.paimon.table.source.Split split = table.newReadBuilder().newScan().plan().splits().get(0); + Map params = createBaseParams(); + params.put("required_fields", "ts"); + params.put("columns_types", toLtz ? "timestamptz(0)" : "datetimev2(0)"); + params.put("time_zone", zone); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + params.put(SERIALIZED_TABLE, Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(table))); + params.put("paimon_split", Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(split))); + params.put("paimon_predicate", Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(Collections.emptyList()))); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(values.length, scanner.getTable().getNumRows()); + Object[] expected = Arrays.stream(values).map(value -> toLtz + ? value.atZone(java.time.ZoneId.of(zone)).withZoneSameInstant(java.time.ZoneOffset.UTC) + .toLocalDateTime().withNano(0) + : value.atZone(java.time.ZoneOffset.UTC) + .withZoneSameInstant(java.time.ZoneId.of(zone)).toLocalDateTime().withNano(0)) + .toArray(); + Assertions.assertArrayEquals(expected, + scanner.getTable().getColumn(0).getObjectColumn(0, values.length), + "SDK zone conversion must be preserved: " + zone + ", toLtz=" + toLtz); + } finally { + scanner.close(); + } + } + } + } finally { + java.util.TimeZone.setDefault(originalZone); + } + } + + @Test + public void narrowedOrcNanosecondsDoNotRoundAcrossSecondBoundary() throws Exception { + org.apache.doris.jni.spi.utils.OffHeap.setTesting(); + try (org.apache.paimon.catalog.Catalog catalog = new org.apache.paimon.catalog.FileSystemCatalog( + org.apache.paimon.fs.local.LocalFileIO.create(), + new org.apache.paimon.fs.Path(temporaryFolder.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "nanoseconds"); + catalog.createTable(id, org.apache.paimon.schema.Schema.newBuilder() + .column("ts", DataTypes.TIMESTAMP(9)).column("ltz", new LocalZonedTimestampType(9)) + .option("file.format", "orc") + .option(org.apache.paimon.format.OrcOptions.ORC_TIMESTAMP_LTZ_LEGACY_TYPE.key(), "false") + .build(), false); + Table table = catalog.getTable(id); + org.apache.paimon.table.sink.BatchWriteBuilder writer = table.newBatchWriteBuilder(); + java.time.LocalDateTime[] values = { + java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1, 999_999_500), + java.time.LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999_999_500)}; + // Paimon's nonlegacy ORC LTZ reader already shifts negative fractional epochs before evolution. + java.time.LocalDateTime ltzValue = values[0]; + try (org.apache.paimon.table.sink.BatchTableWrite write = writer.newWrite()) { + for (java.time.LocalDateTime value : values) { + org.apache.paimon.data.Timestamp timestamp = + org.apache.paimon.data.Timestamp.fromLocalDateTime(value); + write.write(org.apache.paimon.data.GenericRow.of(timestamp, + org.apache.paimon.data.Timestamp.fromLocalDateTime(ltzValue))); + } + try (org.apache.paimon.table.sink.BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + try (RecordReader rawReader = table.newReadBuilder().newRead().createReader( + table.newReadBuilder().newScan().plan().splits().get(0))) { + RecordReader.RecordIterator batch = rawReader.readBatch(); + try { + InternalRow first = batch.next(); + Assertions.assertEquals(values[0], first.getTimestamp(0, 9).toLocalDateTime()); + InternalRow second = batch.next(); + Assertions.assertEquals(values[1], second.getTimestamp(0, 9).toLocalDateTime(), + "check the original ORC p9 file before schema evolution"); + Assertions.assertEquals(ltzValue, second.getTimestamp(1, 9).toLocalDateTime(), + "check the original ORC LTZ p9 file before schema evolution"); + } finally { + batch.releaseBatch(); + } + } + catalog.alterTable(id, Arrays.asList( + org.apache.paimon.schema.SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(0), true), + org.apache.paimon.schema.SchemaChange.updateColumnType("ltz", new LocalZonedTimestampType(0), true)), + false); + table = catalog.getTable(id); + org.apache.paimon.table.source.Split split = table.newReadBuilder().newScan().plan().splits().get(0); + Map params = createBaseParams(); + params.put("required_fields", "ts,ltz"); + params.put("columns_types", "datetimev2(0)#timestamptz(0)"); + params.put("time_zone", "UTC"); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + params.put(SERIALIZED_TABLE, Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(table))); + params.put("paimon_split", Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(split))); + params.put("paimon_predicate", Base64.getUrlEncoder().withoutPadding().encodeToString( + InstantiationUtil.serializeObject(Collections.emptyList()))); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(2, scanner.getTable().getNumRows()); + for (int column = 0; column < 2; column++) { + Object[] actual = scanner.getTable().getColumn(column).getObjectColumn(0, 2); + Object[] expected = column == 0 + ? new Object[] {values[0].withNano(0), values[1].withNano(0)} + : new Object[] {ltzValue.withNano(0), ltzValue.withNano(0)}; + Assertions.assertArrayEquals(expected, actual, + "column " + column); + } + } finally { + scanner.close(); + } + } + } + @Test public void legacyOrcTimestampAndFileMetadataShareOneRead() throws Exception { org.apache.doris.jni.spi.utils.OffHeap.setTesting(); @@ -116,6 +272,7 @@ public void legacyOrcTimestampAndFileMetadataShareOneRead() throws Exception { Map params = createBaseParams(); params.put("required_fields", "__paimon_file_path,event_time,__paimon_row_index"); params.put("columns_types", "string#timestamptz(6)#bigint"); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "false"); params.put(SERIALIZED_TABLE, Base64.getUrlEncoder().withoutPadding().encodeToString( InstantiationUtil.serializeObject(table))); params.put("paimon_split", Base64.getUrlEncoder().withoutPadding().encodeToString( @@ -129,6 +286,10 @@ public void legacyOrcTimestampAndFileMetadataShareOneRead() throws Exception { PaimonJniScanner scanner = new PaimonJniScanner(16, new HashMap<>(params)); try { scanner.open(); + Field readTypes = PaimonJniScanner.class.getDeclaredField("paimonDataTypeList"); + readTypes.setAccessible(true); + Assertions.assertEquals(Collections.singletonList(new LocalZonedTimestampType(6)), + readTypes.get(scanner), "unchanged timestamps must not widen to p9"); Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); int count = filtered ? 1 : 2; Assertions.assertEquals(count, scanner.getTable().getNumRows()); @@ -155,6 +316,44 @@ public void testConstructorAcceptsEmptyProjection() { new PaimonJniScanner(128, createBaseParams()); } + @Test + public void testSafeTimestampReadTypeWidensTimestampColumns() { + RowType nestedType = RowType.of( + new DataType[] {new TimestampType(4)}, new String[] {"nestedTs"}); + RowType tableType = RowType.of( + new DataType[] {new TimestampType(4), new LocalZonedTimestampType(6), + new ArrayType(new TimestampType(5)), nestedType, + new MapType(new TimestampType(4), new LocalZonedTimestampType(6))}, + new String[] {"ts", "ltz", "array", "row", "map"}); + + RowType readType = PaimonJniScanner.createSafeTimestampReadType(tableType); + + Assertions.assertEquals(9, ((TimestampType) readType.getTypeAt(0)).getPrecision()); + Assertions.assertEquals(9, ((LocalZonedTimestampType) readType.getTypeAt(1)).getPrecision()); + Assertions.assertEquals(9, ((TimestampType) ((ArrayType) readType.getTypeAt(2)).getElementType()) + .getPrecision()); + Assertions.assertEquals(9, ((TimestampType) ((RowType) readType.getTypeAt(3)).getTypeAt(0)) + .getPrecision()); + MapType readMapType = (MapType) readType.getTypeAt(4); + Assertions.assertEquals(4, ((TimestampType) readMapType.getKeyType()).getPrecision()); + Assertions.assertEquals(9, ((LocalZonedTimestampType) readMapType.getValueType()).getPrecision()); + + RowType projectedReadType = readType.project(new int[] {1}); + Assertions.assertEquals("ltz", projectedReadType.getFieldNames().get(0)); + Assertions.assertEquals(9, ((LocalZonedTimestampType) projectedReadType.getTypeAt(0)).getPrecision()); + } + + @Test + public void testNestedTimestampProjectionActivatesRepair() { + ColumnType nested = ColumnType.parseType("payload", "struct"); + ColumnType map = ColumnType.parseType("entries", "map"); + + Assertions.assertTrue(PaimonJniScanner.containsTimestampType(nested)); + Assertions.assertTrue(PaimonJniScanner.containsTimestampType(map)); + Assertions.assertFalse(PaimonJniScanner.containsTimestampType( + ColumnType.parseType("payload", "struct"))); + } + @Test public void testConstructorRejectsMissingOrEmptyTableCacheKey() { Map params = createBaseParams(); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonTimestampRepairReaderTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonTimestampRepairReaderTest.java new file mode 100644 index 00000000000000..9185f3b283c8a4 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonTimestampRepairReaderTest.java @@ -0,0 +1,807 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.doris.jni.spi.utils.OffHeap; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.predicate.Transform; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.ChainGroupReadTable; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ChainSplit; +import org.apache.paimon.table.source.QueryAuthSplit; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PaimonTimestampRepairReaderTest { + @TempDir + Path warehouse; + + @Test + public void legacyProducerRetainsSafePredicateAndHistoricalTimestamp() throws Exception { + checkReader("legacy", new Object[] { + LocalDateTime.of(1969, 12, 31, 23, 59, 59), LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void hiddenIntegerPolicyColumnDoesNotResetTimestampReadType() throws Exception { + checkReader("integer_policy", new Object[] { + LocalDateTime.of(1969, 12, 31, 23, 59, 59), LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void timestampPolicyRunsAfterRepairAndRejectsNonmatchingRows() throws Exception { + checkReader("timestamp_policy", new Object[] {LocalDateTime.of(2025, 1, 1, 0, 0, 1), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void auditLogHistoricalProjectionRetainsNegativeEpoch() throws Exception { + checkReader("audit_log", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1), LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void auditLogSystemColumnAndSerializedIntegerPolicyRetainPrecision() throws Exception { + checkReader("audit_log_policy", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void fallbackTimestampPolicyRunsAfterPrecisionRepair() throws Exception { + checkReader("fallback_policy", new Object[] {LocalDateTime.of(2025, 1, 1, 0, 0, 1), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void fallbackRenamedHiddenPolicyColumnUsesSelectedBranchSchema() throws Exception { + checkReader("fallback_renamed_policy", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void primaryOrderingRetainsRenamedHiddenFallbackPolicyColumn() throws Exception { + checkReader("fallback_primary_renamed_policy", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void auditLogFallbackUsesRenamedHiddenPolicyColumn() throws Exception { + checkReader("fallback_audit_renamed_policy", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void auditLogPrimaryOrderingUsesRenamedHiddenPolicyColumn() throws Exception { + checkReader("fallback_primary_audit_renamed_policy", new Object[] { + LocalDateTime.of(1969, 12, 31, 23, 59, 59), LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void fallbackDataSplitWithoutPolicyRemainsAReaderLeaf() throws Exception { + checkReader("fallback_plain", new Object[] {LocalDateTime.of(1969, 12, 31, 23, 59, 59), + LocalDateTime.of(2025, 1, 1, 0, 0, 1), LocalDateTime.of(2025, 1, 1, 0, 0, 1)}); + } + + @Test + public void fallbackHistoricalOrcUsesSelectedBranchFieldIds() throws Exception { + checkFallbackBranchFieldIds(false, null, "true"); + } + + @Test + public void fallbackRenamedFieldsRetainHiddenPolicyAndFieldIds() throws Exception { + checkFallbackBranchFieldIds(true, null, "true"); + } + + @Test + public void auditLogFallbackRetainsSystemColumnsAndBranchFieldIds() throws Exception { + checkFallbackBranchFieldIds(true, "audit_log", "true"); + } + + @Test + public void legacyFallbackHistoricalOrcUsesSelectedBranchFieldIds() throws Exception { + checkFallbackBranchFieldIds(false, null, null); + } + + @Test + public void fallbackWithoutRepairKeepsSdkProjection() throws Exception { + checkFallbackBranchFieldIds(false, null, "false"); + } + + @Test + public void binlogFallbackRetainsArrayWrapperAndBranchFieldIds() throws Exception { + checkFallbackBranchFieldIds(false, "binlog", "true"); + } + + @Test + public void legacyBinlogFallbackRetainsArrayWrapperAndBranchFieldIds() throws Exception { + checkFallbackBranchFieldIds(false, "binlog", null); + } + + @Test + public void binlogWithoutRepairKeepsSdkArrayProjection() throws Exception { + checkFallbackBranchFieldIds(false, "binlog", "false"); + } + + @Test + public void chainAuditLogUsesDeltaReaderFieldIds() throws Exception { + OffHeap.setTesting(); + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier mainId = Identifier.create("db", "main"); + Identifier snapshotId = Identifier.create("db", "snapshot"); + Identifier deltaId = Identifier.create("db", "delta"); + for (Identifier id : Arrays.asList(mainId, snapshotId, deltaId)) { + catalog.createTable(id, Schema.newBuilder().column("pt", DataTypes.INT()) + .column("id", DataTypes.INT()).column("ts", DataTypes.TIMESTAMP(6)) + .primaryKey("pt", "id").partitionKeys("pt") + .option("bucket", "1").option("file.format", "parquet").build(), false); + } + catalog.alterTable(snapshotId, Collections.singletonList(SchemaChange.dropColumn("ts")), false); + catalog.alterTable(snapshotId, Collections.singletonList( + SchemaChange.addColumn("ts", DataTypes.TIMESTAMP(6))), false); + LocalDateTime second = LocalDateTime.of(2025, 1, 1, 0, 0, 1); + writeRows(catalog.getTable(deltaId), + GenericRow.of(1, 1, Timestamp.fromLocalDateTime(second)), + GenericRow.of(1, 2, Timestamp.fromLocalDateTime(second.withNano(600000000))), + GenericRow.of(1, 3, null)); + for (Identifier id : Arrays.asList(mainId, snapshotId, deltaId)) { + catalog.alterTable(id, Collections.singletonList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(0), true)), false); + } + FileStoreTable main = (FileStoreTable) catalog.getTable(mainId); + FileStoreTable snapshot = (FileStoreTable) catalog.getTable(snapshotId); + FileStoreTable delta = (FileStoreTable) catalog.getTable(deltaId); + Assertions.assertNotEquals(snapshot.rowType().getFields().get(2).id(), + delta.rowType().getFields().get(2).id()); + Map chainOptions = new HashMap<>(); + chainOptions.put(CoreOptions.SCAN_FALLBACK_SNAPSHOT_BRANCH.key(), "snapshot"); + chainOptions.put(CoreOptions.SCAN_FALLBACK_DELTA_BRANCH.key(), "main"); + FileStoreTable chain = new FallbackReadFileStoreTable(main, + new ChainGroupReadTable(snapshot.copy(chainOptions), delta.copy(chainOptions)), true); + Table audit = org.apache.paimon.table.system.SystemTableLoader.load("audit_log", chain); + List splits = audit.newReadBuilder().newScan().plan().splits(); + Assertions.assertEquals(1, splits.size()); + Split split = splits.get(0); + Assertions.assertTrue(split instanceof FallbackReadFileStoreTable.FallbackSplit); + Assertions.assertTrue(((FallbackReadFileStoreTable.FallbackSplit) split).wrapped() instanceof ChainSplit); + Map params = new HashMap<>(); + params.put("required_fields", "rowkind,id,ts"); + params.put("columns_types", "string#int#datetimev2(0)"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "chain-audit-" + warehouse); + params.put("serialized_table", encode(audit)); + params.put("paimon_split", encode(split)); + params.put("paimon_predicate", encode(Collections.emptyList())); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(3, scanner.getTable().getNumRows()); + Assertions.assertArrayEquals(new Object[] {"+I", "+I", "+I"}, + scanner.getTable().getColumn(0).getObjectColumn(0, 3)); + Assertions.assertArrayEquals(new Object[] {1, 2, 3}, + scanner.getTable().getColumn(1).getObjectColumn(0, 3)); + Assertions.assertArrayEquals(new Object[] {second, second, null}, + scanner.getTable().getColumn(2).getObjectColumn(0, 3)); + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + } + + private void checkFallbackBranchFieldIds(boolean renamedPolicy, String systemTable, String repairOption) + throws Exception { + OffHeap.setTesting(); + boolean audit = systemTable != null; + boolean binlog = "binlog".equals(systemTable); + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier mainId = Identifier.create("db", "main"); + Identifier otherId = Identifier.create("db", "other"); + catalog.createTable(mainId, Schema.newBuilder().column("pt", DataTypes.INT()) + .column("id", DataTypes.INT()).column("allowed", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(0)) + .partitionKeys("pt").option("file.format", "orc").build(), false); + if (!"false".equals(repairOption)) { + catalog.alterTable(mainId, Collections.singletonList(SchemaChange.dropColumn("ts")), false); + catalog.alterTable(mainId, Collections.singletonList( + SchemaChange.addColumn("ts", DataTypes.TIMESTAMP(0))), false); + } + catalog.createTable(otherId, Schema.newBuilder().column("pt", DataTypes.INT()) + .column("id", DataTypes.INT()).column("allowed", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(binlog ? 6 : 9)) + .partitionKeys("pt").option("file.format", "orc").build(), false); + // Match schema versions, but not the independently allocated timestamp field IDs. + catalog.alterTable(otherId, Collections.singletonList(SchemaChange.setOption("test.option", "1")), + false); + LocalDateTime second = LocalDateTime.of(2025, 1, 1, 0, 0, 1); + writeRows(catalog.getTable(mainId), GenericRow.of(0, 0, 7, Timestamp.fromLocalDateTime(second))); + writeRows(catalog.getTable(otherId), + GenericRow.of(1, 1, 7, Timestamp.fromLocalDateTime(second)), + GenericRow.of(1, 2, 7, Timestamp.fromLocalDateTime( + second.withNano(binlog ? 600000000 : 999999500))), + GenericRow.of(1, 3, 7, null)); + if (renamedPolicy) { + writeRows(catalog.getTable(otherId), GenericRow.of(1, 4, 8, Timestamp.fromLocalDateTime(second))); + } + catalog.alterTable(otherId, Collections.singletonList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(0), true)), false); + FileStoreTable main = (FileStoreTable) catalog.getTable(mainId); + FileStoreTable other = (FileStoreTable) catalog.getTable(otherId); + if (renamedPolicy) { + catalog.alterTable(otherId, Arrays.asList(SchemaChange.renameColumn("ts", "branch_ts"), + SchemaChange.renameColumn("allowed", "branch_allowed")), false); + other = (FileStoreTable) catalog.getTable(otherId); + } + Assertions.assertEquals(!"false".equals(repairOption), main.rowType().getFields().get(3).id() + != other.rowType().getFields().get(3).id()); + Map expected = new HashMap<>(); + expected.put(0, second); + expected.put(1, second); + expected.put(2, second); + expected.put(3, null); + for (boolean wrappedFirst : new boolean[] {true, false}) { + FileStoreTable wrapped = wrappedFirst ? main : main.copy(Collections.singletonMap( + CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")); + Table pair = new FallbackReadFileStoreTable(wrapped, other, wrappedFirst); + if (audit) { + pair = org.apache.paimon.table.system.SystemTableLoader.load(systemTable, (FileStoreTable) pair); + } + List splits = pair.newReadBuilder().newScan().plan().splits(); + if (renamedPolicy) { + // FileSystemCatalog has no policy provider. Attach serialized policy results below, + // after planning, just as the existing historical-reader policy tests do. + pair = new FallbackReadFileStoreTable(wrapped, + other.copy(Collections.singletonMap("query-auth.enabled", "true")), wrappedFirst); + if (audit) { + pair = org.apache.paimon.table.system.SystemTableLoader.load(systemTable, (FileStoreTable) pair); + } + } + Map actual = new HashMap<>(); + for (Split split : splits) { + if (renamedPolicy && ((FallbackReadFileStoreTable.FallbackSplit) split).isFallback() + == wrappedFirst) { + PredicateBuilder policy = new PredicateBuilder(other.rowType()); + Split policySplit = new QueryAuthSplit( + ((FallbackReadFileStoreTable.FallbackSplit) split).wrapped(), + new TableQueryAuthResult(Collections.singletonList(JsonSerdeUtil.toJson( + policy.equal(policy.indexOf("branch_allowed"), 7))), + Collections.emptyMap())); + split = wrappedFirst ? FallbackReadFileStoreTable.toFallbackSplit(policySplit, true) + : policySplit; + } + Map params = new HashMap<>(); + params.put("required_fields", audit ? "rowkind,ts,id" : "ts,id"); + params.put("columns_types", binlog ? "string#array#array" + : audit ? "string#datetimev2(0)#int" : "datetimev2(0)#int"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "fallback-field-ids-" + warehouse + wrappedFirst); + params.put("serialized_table", encode(pair)); + params.put("paimon_split", encode(split)); + params.put("paimon_predicate", encode(Collections.emptyList())); + if (repairOption != null) { + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, repairOption); + } + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0 && scanner.getTable().getNumRows() > 0) { + int count = scanner.getTable().getNumRows(); + Object[] timestamps = scanner.getTable().getColumn(audit ? 1 : 0).getObjectColumn(0, count); + Object[] ids = scanner.getTable().getColumn(audit ? 2 : 1).getObjectColumn(0, count); + for (int i = 0; i < count; i++) { + Object id = ids[i]; + Object timestamp = timestamps[i]; + if (binlog) { + Assertions.assertEquals(1, ((List) id).size()); + Assertions.assertEquals(1, ((List) timestamp).size()); + id = ((List) id).get(0); + timestamp = ((List) timestamp).get(0); + } + if (audit) { + Assertions.assertEquals("+I", scanner.getTable().getColumn(0) + .getObjectColumn(0, count)[i]); + } + Assertions.assertFalse(actual.containsKey((Integer) id)); + actual.put((Integer) id, timestamp); + } + } + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + Assertions.assertEquals(expected, actual, "wrappedFirst=" + wrappedFirst); + if (repairOption == null && !audit) { + checkSnapshotMetadata(pair); + } + } + } + } + + private void checkSnapshotMetadata(Table pair) throws Exception { + Table snapshots = org.apache.paimon.table.system.SystemTableLoader.load("snapshots", (FileStoreTable) pair); + Map params = new HashMap<>(); + params.put("required_fields", "snapshot_id,commit_time"); + params.put("columns_types", "bigint#datetimev2(3)"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "fallback-snapshots-" + warehouse); + params.put("serialized_table", encode(snapshots)); + params.put("paimon_split", encode(snapshots.newReadBuilder().newScan().plan().splits().get(0))); + params.put("paimon_predicate", encode(Collections.emptyList())); + // An older FE sends no repair option for metadata tables either. + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(1, scanner.getTable().getNumRows()); + Assertions.assertEquals(1L, scanner.getTable().getColumn(0).getObjectColumn(0, 1)[0]); + Assertions.assertNotNull(scanner.getTable().getColumn(1).getObjectColumn(0, 1)[0]); + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + + private static void writeRows(Table table, GenericRow... rows) throws Exception { + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + for (GenericRow row : rows) { + write.write(row); + } + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + } + + @Test + public void nanosecondOutputsRetainLogicalPrecisionWithDatetimeSibling() throws Exception { + for (int precision : new int[] {7, 8}) { + checkNanosecondReader(precision, false); + checkNanosecondReader(precision, true); + } + } + + private void checkNanosecondReader(int precision, boolean withSibling) throws Exception { + OffHeap.setTesting(); + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", true); + Identifier id = Identifier.create("db", "nano_" + precision + "_" + withSibling); + catalog.createTable(id, Schema.newBuilder().column("ts", DataTypes.TIMESTAMP(9)) + .column("nested", RowType.builder().field("ts", DataTypes.TIMESTAMP(9)).build()) + .column("plain_ts", DataTypes.TIMESTAMP(6)).option("file.format", "parquet").build(), false); + Table table = catalog.getTable(id); + LocalDateTime value = LocalDateTime.of(2025, 1, 1, 0, 0, 1, 123456789); + Timestamp timestamp = Timestamp.fromLocalDateTime(value); + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + write.write(GenericRow.of(timestamp, GenericRow.of(timestamp), + Timestamp.fromLocalDateTime(value.withNano(123456000)))); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(id, Arrays.asList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(precision), true), + SchemaChange.updateColumnType(new String[] {"nested", "ts"}, + DataTypes.TIMESTAMP(precision), true)), false); + table = catalog.getTable(id); + Map params = new HashMap<>(); + params.put("required_fields", withSibling ? "ts,nested,plain_ts" : "ts"); + params.put("columns_types", withSibling + ? "timestamp_ns#struct#datetimev2(6)" : "timestamp_ns"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "nano-reader-" + id + warehouse); + params.put("serialized_table", encode(table)); + params.put("paimon_split", encode(table.newReadBuilder().newScan().plan().splits().get(0))); + params.put("paimon_predicate", encode(Collections.emptyList())); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(1, scanner.getTable().getNumRows()); + LocalDateTime expected = value.withNano(precision == 7 ? 123456700 : 123456780); + Assertions.assertEquals(expected, scanner.getTable().getColumn(0).getObjectColumn(0, 1)[0]); + if (withSibling) { + Assertions.assertEquals(Collections.singletonMap("ts", expected), + scanner.getTable().getColumn(1).getObjectColumn(0, 1)[0]); + Assertions.assertEquals(value.withNano(123456000), + scanner.getTable().getColumn(2).getObjectColumn(0, 1)[0]); + } + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + } + + @Test + public void integerPolicyAndMaskRemainEnforcedAfterRepair() throws Exception { + checkReader("masking_policy", new Object[] {LocalDateTime.of(2020, 1, 1, 0, 0), + LocalDateTime.of(2020, 1, 1, 0, 0)}); + } + + @Test + public void hiddenTimestampPolicyActivatesRepairForIntegerProjection() throws Exception { + checkReader("hidden_timestamp_policy", new Object[] {7}); + } + + @Test + public void fallbackHiddenTimestampPolicyRepairsLeadingZeroFraction() throws Exception { + OffHeap.setTesting(); + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier mainId = Identifier.create("db", "main"); + Identifier otherId = Identifier.create("db", "other"); + catalog.createTable(mainId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(4)).partitionKeys("id").build(), false); + catalog.createTable(otherId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(6)).partitionKeys("id") + .option("file.format", "parquet").build(), false); + Table other = catalog.getTable(otherId); + BatchWriteBuilder writer = other.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + write.write(GenericRow.of(7, Timestamp.fromLocalDateTime( + LocalDateTime.of(2025, 1, 1, 0, 0, 1, 1000)))); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(otherId, Collections.singletonList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(4), true)), false); + other = catalog.getTable(otherId); + PredicateBuilder builder = new PredicateBuilder(other.rowType()); + Predicate policy = builder.equal(builder.indexOf("ts"), + Timestamp.fromLocalDateTime(LocalDateTime.of(2025, 1, 1, 0, 0, 1))); + Split split = FallbackReadFileStoreTable.toFallbackSplit(new QueryAuthSplit( + other.newReadBuilder().newScan().plan().splits().get(0), + new TypedPolicyAuth(policy, Collections.emptyMap())), true); + Table pair = new FallbackReadFileStoreTable((FileStoreTable) catalog.getTable(mainId), + ((FileStoreTable) other).copy(Collections.singletonMap("query-auth.enabled", "true")), true); + Map params = new HashMap<>(); + params.put("required_fields", "id"); + params.put("columns_types", "int"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "fallback-hidden-policy-" + warehouse); + params.put("serialized_table", encode(pair)); + params.put("paimon_split", encode(split)); + params.put("paimon_predicate", encode(Collections.emptyList())); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(1, scanner.getTable().getNumRows()); + Assertions.assertArrayEquals(new Object[] {7}, + scanner.getTable().getColumn(0).getObjectColumn(0, 1)); + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + } + + @Test + public void nestedPolicyRetainsUnprojectedSiblingAndRestoresOutputShape() throws Exception { + checkReader("nested_policy", new Object[] {Collections.singletonMap("visible", 7)}); + } + + @Test + public void policyMaterializationRepairsNestedValuesWithoutChangingMapKeys() { + RowType current = RowType.builder().field("array", DataTypes.ARRAY(DataTypes.TIMESTAMP(0))) + .field("map", DataTypes.MAP(DataTypes.TIMESTAMP(6), DataTypes.TIMESTAMP(0))) + .field("row", RowType.builder().field("ts", DataTypes.TIMESTAMP(0)).build()).build(); + RowType widened = PaimonJniScanner.createSafeTimestampReadType(current); + Timestamp negative = Timestamp.fromLocalDateTime(LocalDateTime.of(1969, 12, 31, 23, 59, 59, 600000000)); + Timestamp key = Timestamp.fromLocalDateTime(LocalDateTime.of(2025, 1, 1, 0, 0, 1, 1000)); + InternalRow repaired = PaimonTimestampQueryAuth.repairRow(GenericRow.of( + new org.apache.paimon.data.GenericArray(new Object[] {negative, null}), + new org.apache.paimon.data.GenericMap(Collections.singletonMap(key, negative)), + GenericRow.of(key)), widened, current); + Timestamp expectedNegative = Timestamp.fromLocalDateTime(LocalDateTime.of(1969, 12, 31, 23, 59, 59)); + Assertions.assertEquals(expectedNegative, repaired.getArray(0).getTimestamp(0, 0)); + Assertions.assertTrue(repaired.getArray(0).isNullAt(1)); + Assertions.assertSame(key, repaired.getMap(1).keyArray().getTimestamp(0, 6)); + Assertions.assertEquals(expectedNegative, repaired.getMap(1).valueArray().getTimestamp(0, 0)); + Assertions.assertEquals(Timestamp.fromLocalDateTime(LocalDateTime.of(2025, 1, 1, 0, 0, 1)), + repaired.getRow(2, 1).getTimestamp(0, 0)); + } + + private void checkReader(String mode, Object[] expected) throws Exception { + OffHeap.setTesting(); + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "events"); + boolean nested = mode.equals("nested_policy"); + boolean renamedPolicy = mode.startsWith("fallback_") && mode.endsWith("renamed_policy"); + boolean auditPolicy = mode.equals("audit_log_policy") || mode.contains("_audit_"); + Schema.Builder schema = Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(6)).option("file.format", "parquet"); + if (nested) { + schema.column("payload", RowType.builder().field("visible", DataTypes.INT()) + .field("hidden", DataTypes.TIMESTAMP(6)).build()); + } + catalog.createTable(id, schema.build(), false); + Table table = catalog.getTable(id); + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + Timestamp negative = Timestamp.fromLocalDateTime(LocalDateTime.of(1969, 12, 31, 23, 59, 59, 600000000)); + Timestamp positive = Timestamp.fromLocalDateTime(LocalDateTime.of(2025, 1, 1, 0, 0, 1, 600000000)); + try (BatchTableWrite write = writer.newWrite()) { + write.write(nested ? GenericRow.of(7, negative, GenericRow.of(7, negative)) + : GenericRow.of(7, negative)); + write.write(nested ? GenericRow.of(7, positive, GenericRow.of(7, positive)) + : GenericRow.of(7, positive)); + write.write(nested ? GenericRow.of(8, positive, GenericRow.of(8, positive)) + : GenericRow.of(8, positive)); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + List changes = new java.util.ArrayList<>(); + changes.add(SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(0), true)); + if (renamedPolicy) { + changes.add(SchemaChange.renameColumn("id", "renamed_id")); + } + if (nested) { + changes.add(SchemaChange.updateColumnType(new String[] {"payload", "hidden"}, + DataTypes.TIMESTAMP(0), true)); + } + catalog.alterTable(id, changes, false); + table = catalog.getTable(id); + if (mode.startsWith("audit_log")) { + table = org.apache.paimon.table.system.SystemTableLoader.load("audit_log", + (org.apache.paimon.table.FileStoreTable) table); + } + Split split = table.newReadBuilder().newScan().plan().splits().get(0); + PredicateBuilder builder = new PredicateBuilder(table.rowType()); + Predicate integer = builder.equal(builder.indexOf( + renamedPolicy ? "renamed_id" : "id"), 7); + Predicate timestamp = builder.equal(builder.indexOf("ts"), + Timestamp.fromLocalDateTime(mode.equals("hidden_timestamp_policy") + ? LocalDateTime.of(1969, 12, 31, 23, 59, 59) : LocalDateTime.of(2025, 1, 1, 0, 0, 1))); + Map params = new HashMap<>(); + params.put("required_fields", "ts"); + params.put("columns_types", "datetimev2(0)"); + params.put("time_zone", "UTC"); + params.put("serialized_table_cache_key", "repair-reader-" + warehouse); + params.put("serialized_table", encode(table)); + if (auditPolicy) { + params.put("required_fields", "rowkind,ts"); + params.put("columns_types", "string#datetimev2(0)"); + } else if (mode.equals("hidden_timestamp_policy")) { + params.put("required_fields", "id"); + params.put("columns_types", "int"); + } else if (nested) { + params.put("required_fields", "payload"); + params.put("columns_types", "struct"); + } + if (mode.equals("legacy")) { + params.put("required_fields", "id,ts"); + params.put("columns_types", "int#datetimev2(0)"); + params.put("paimon_predicate", encode( + Collections.singletonList(PredicateBuilder.and(integer, timestamp)))); + } else if (!mode.equals("audit_log") && !mode.equals("fallback_plain")) { + Predicate policy = mode.equals("timestamp_policy") || mode.equals("hidden_timestamp_policy") + || mode.equals("fallback_policy") + ? timestamp : integer; + if (nested) { + policy = builder.equal(new HiddenTimestamp(Collections.singletonList(new FieldRef( + builder.indexOf("payload"), "payload", + table.rowType().getTypeAt(builder.indexOf("payload"))))), + Timestamp.fromLocalDateTime(LocalDateTime.of(1969, 12, 31, 23, 59, 59))); + } + TableQueryAuthResult auth = mode.equals("integer_policy") || mode.equals("audit_log_policy") + || renamedPolicy + ? new TableQueryAuthResult( + Collections.singletonList(JsonSerdeUtil.toJson(policy)), Collections.emptyMap()) + : new TypedPolicyAuth(policy, mode.equals("masking_policy") + ? Collections.singletonMap("ts", new TimestampMask(Collections.singletonList( + new FieldRef(1, "ts", DataTypes.TIMESTAMP(0))))) : Collections.emptyMap()); + split = new QueryAuthSplit(split, auth); + params.put("paimon_predicate", encode(Collections.emptyList())); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + } else { + params.put("paimon_predicate", encode(Collections.emptyList())); + params.put("paimon." + PaimonJniScanner.DORIS_TIMESTAMP_PRECISION_REPAIR, "true"); + } + if (mode.startsWith("fallback_")) { + split = FallbackReadFileStoreTable.toFallbackSplit(split, true); + Identifier mainId = Identifier.create("db", "main_events"); + catalog.createTable(mainId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(0)).option("file.format", "parquet").build(), false); + if (mode.startsWith("fallback_primary_")) { + table = new FallbackReadFileStoreTable(((FileStoreTable) table).copy( + Collections.singletonMap(CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")), + (FileStoreTable) catalog.getTable(mainId), false); + } else { + table = new FallbackReadFileStoreTable((FileStoreTable) catalog.getTable(mainId), + (FileStoreTable) table, true); + } + if (auditPolicy) { + table = org.apache.paimon.table.system.SystemTableLoader.load("audit_log", (FileStoreTable) table); + } + params.put("serialized_table", encode(table)); + } + params.put("paimon_split", encode(split)); + PaimonJniScanner scanner = new PaimonJniScanner(16, params); + try { + scanner.open(); + if (mode.equals("legacy")) { + java.lang.reflect.Method predicates = PaimonJniScanner.class.getDeclaredMethod("getPredicates"); + predicates.setAccessible(true); + Assertions.assertEquals(Collections.singletonList(integer), predicates.invoke(scanner)); + } + Assertions.assertNotEquals(0, scanner.getNextBatchMeta()); + Assertions.assertEquals(expected.length, scanner.getTable().getNumRows()); + if (auditPolicy) { + Assertions.assertArrayEquals(new Object[] {"+I", "+I"}, + scanner.getTable().getColumn(0).getObjectColumn(0, expected.length)); + } + Object[] actual = scanner.getTable().getColumn( + mode.equals("legacy") || auditPolicy ? 1 : 0) + .getObjectColumn(0, expected.length); + Arrays.sort(actual); + Assertions.assertArrayEquals(expected, actual, mode); + } finally { + scanner.close(); + PaimonTableCache.clearForTest(); + } + } + } + + private static String encode(Object value) throws Exception { + return Base64.getUrlEncoder().withoutPadding().encodeToString(InstantiationUtil.serializeObject(value)); + } + + private static final class TypedPolicyAuth extends TableQueryAuthResult { + private static final long serialVersionUID = 1L; + private final Predicate policy; + private final Map masking; + + private TypedPolicyAuth(Predicate policy, Map masking) { + super(Collections.emptyList(), Collections.emptyMap()); + this.policy = policy; + this.masking = masking; + } + + @Override + public Predicate extractPredicate() { + // Exercise the SDK policy callback independently of its timestamp JSON codec. + return policy; + } + + @Override + public Map extractColumnMasking() { + return masking; + } + } + + private static final class TimestampMask implements Transform { + private static final long serialVersionUID = 1L; + private final List inputs; + + private TimestampMask(List inputs) { + this.inputs = inputs; + } + + @Override + public String name() { + return "TEST_TIMESTAMP_MASK"; + } + + @Override + public List inputs() { + return inputs; + } + + @Override + public DataType outputType() { + return DataTypes.TIMESTAMP(0); + } + + @Override + public Object transform(InternalRow row) { + FieldRef source = (FieldRef) inputs.get(0); + Assertions.assertEquals(0, row.getTimestamp(source.index(), 0).toLocalDateTime().getNano()); + return Timestamp.fromLocalDateTime(LocalDateTime.of(2020, 1, 1, 0, 0)); + } + + @Override + public Transform copyWithNewInputs(List newInputs) { + return new TimestampMask(newInputs); + } + } + + private static final class HiddenTimestamp implements Transform { + private static final long serialVersionUID = 1L; + private final List inputs; + + private HiddenTimestamp(List inputs) { + this.inputs = inputs; + } + + @Override + public String name() { + return "TEST_HIDDEN_TIMESTAMP"; + } + + @Override + public List inputs() { + return inputs; + } + + @Override + public DataType outputType() { + return DataTypes.TIMESTAMP(0); + } + + @Override + public Object transform(InternalRow row) { + return row.getRow(((FieldRef) inputs.get(0)).index(), 2).getTimestamp(1, 0); + } + + @Override + public Transform copyWithNewInputs(List newInputs) { + return new HiddenTimestamp(newInputs); + } + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPredicateConverter.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPredicateConverter.java index b959a88a12a027..0b7fcae7f48cc3 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPredicateConverter.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPredicateConverter.java @@ -61,9 +61,15 @@ public class PaimonPredicateConverter { private final PredicateBuilder builder; private final List fieldNames; private final List fieldTypes; + private final boolean dropTimestampPredicates; public PaimonPredicateConverter(RowType rowType) { + this(rowType, false); + } + + public PaimonPredicateConverter(RowType rowType, boolean dropTimestampPredicates) { this.builder = new PredicateBuilder(rowType); + this.dropTimestampPredicates = dropTimestampPredicates; this.fieldNames = rowType.getFields().stream() .map(f -> f.name().toLowerCase()) .collect(Collectors.toList()); @@ -148,6 +154,9 @@ private Predicate convertComparison(ConnectorComparison cmp) { return null; } ConnectorColumnRef colRef = (ConnectorColumnRef) left; + if (dropTimestampPredicates && isTimestampColumn(colRef)) { + return null; + } ConnectorLiteral literal = (ConnectorLiteral) right; int idx = fieldNames.indexOf(colRef.getColumnName().toLowerCase()); if (idx < 0) { @@ -195,6 +204,9 @@ private Predicate convertIn(ConnectorIn in) { return null; } ConnectorColumnRef colRef = (ConnectorColumnRef) valueExpr; + if (dropTimestampPredicates && isTimestampColumn(colRef)) { + return null; + } int idx = fieldNames.indexOf(colRef.getColumnName().toLowerCase()); if (idx < 0) { return null; @@ -219,8 +231,11 @@ private Predicate convertIsNull(ConnectorIsNull isNull) { if (!(operand instanceof ConnectorColumnRef)) { return null; } - int idx = fieldNames.indexOf( - ((ConnectorColumnRef) operand).getColumnName().toLowerCase()); + ConnectorColumnRef colRef = (ConnectorColumnRef) operand; + if (dropTimestampPredicates && isTimestampColumn(colRef)) { + return null; + } + int idx = fieldNames.indexOf(colRef.getColumnName().toLowerCase()); if (idx < 0) { return null; } @@ -237,6 +252,9 @@ private Predicate convertLike(ConnectorLike like) { || !(patternExpr instanceof ConnectorLiteral)) { return null; } + if (dropTimestampPredicates && isTimestampColumn((ConnectorColumnRef) valueExpr)) { + return null; + } int idx = fieldNames.indexOf( ((ConnectorColumnRef) valueExpr).getColumnName().toLowerCase()); if (idx < 0) { @@ -250,6 +268,16 @@ private Predicate convertLike(ConnectorLike like) { return builder.startsWith(idx, BinaryString.fromString(prefix)); } + private boolean isTimestampColumn(ConnectorColumnRef column) { + int idx = fieldNames.indexOf(column.getColumnName().toLowerCase()); + if (idx < 0) { + return false; + } + DataTypeRoot root = fieldTypes.get(idx).getTypeRoot(); + return root == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE + || root == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE; + } + /** * The literal prefix a Doris LIKE pattern is exactly equivalent to, or {@code null} when no such * proof exists. diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 692170397f4d78..5e6885db66a4e1 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -232,6 +232,7 @@ public ConnectorColumnCategory classifyColumn(String columnName) { "doris.scan.manifest.parallelism-cap"; private static final String DORIS_SERIALIZED_SYSTEM_SOURCE = "doris.serialized-system-source"; private static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type"; + static final String DORIS_TIMESTAMP_PRECISION_REPAIR = "doris.timestamp-precision-repair"; private final PaimonCatalogProperties catalogProps; private final PaimonCatalogOps catalogOps; @@ -824,8 +825,13 @@ private List planScanInternal( // Build predicates from filter expression RowType rowType = table.rowType(); List predicates = Collections.emptyList(); + // Paimon can cast historical timestamp statistics with the current precision while planning + // splits. Keep these predicates as Doris residuals so an old file cannot be pruned before JNI + // widens and repairs its timestamp values. if (filter.isPresent()) { - PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType); + PaimonPredicateConverter converter = new PaimonPredicateConverter( + rowType, filterReadsTimestamp(rowType, filter.get()) + && requiresTimestampPrecisionRepair(session, table, paimonHandle)); predicates = converter.convert(filter.get()); } @@ -975,6 +981,7 @@ && hasTrustworthyLimitAccounting(table) // Schema IDs are branch-local, even when both branches expose the same current row type. Map> legacyOrcTimestampSchemas = new java.util.IdentityHashMap<>(); + Map> crossFamilyTimestampSchemas = new java.util.IdentityHashMap<>(); Set readFieldIds = scanReadFieldIds(rowType, columns, filter); // Native INT96/ORC materialization currently narrows these values to microseconds. // JNI retains the source precision, including nested and filter-only timestamp fields. @@ -1003,10 +1010,14 @@ && hasTrustworthyLimitAccounting(table) isForceJniScannerEnabled(session), hasVariantProjection, physicalVariantSchemaIds, optRawFiles); boolean rustEligible = rustReaderSelector != null && rustReaderSelector.canRead(dataSplit); boolean legacyOrcTimestamp = false; - if (nativeEligible && (!rustEligible || aggregatePushdown)) { + if (nativeEligible && containsTimestampType(rowType) && (!rustEligible || aggregatePushdown)) { FileStoreTable splitTable = legacyOrcSplitTable((FileStoreTable) legacyOrcSchemaTable.get(), dataSplit); - legacyOrcTimestamp = requiresLegacyOrcTimestampReader(splitTable, optRawFiles, readFieldIds, - legacyOrcTimestampSchemas.computeIfAbsent(splitTable, ignored -> new HashMap<>())); + Set splitReadFieldIds = branchReadFieldIds(rowType, splitTable.rowType(), readFieldIds); + legacyOrcTimestamp = withSchemaHistoryAuthentication(() -> + requiresCrossFamilyTimestampReader(splitTable, optRawFiles, splitReadFieldIds, + crossFamilyTimestampSchemas.computeIfAbsent(splitTable, ignored -> new HashMap<>())) + || requiresOrcTimestampReader(splitTable, optRawFiles, splitReadFieldIds, + legacyOrcTimestampSchemas.computeIfAbsent(splitTable, ignored -> new HashMap<>()))); } // Only the native reader can answer COUNT(column) and MIN/MAX from file footers. // Keep ordinary compatible scans on Rust, but preserve native aggregate pushdown. @@ -1241,15 +1252,26 @@ public Map getScanNodeProperties( // ("encodedStr is null"). Mirrors legacy PaimonScanNode.createScanRangeLocations, which always called // setPaimonPredicate(encodeObjectToString(predicates)) regardless of whether predicates was empty. List predicates = Collections.emptyList(); + // The same predicate must stay out of the BE Paimon reader: JNI materializes and repairs the + // value first, then the generic scan path evaluates the residual against the repaired column. if (filter.isPresent()) { RowType rowType = table.rowType(); - PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType); + PaimonPredicateConverter converter = new PaimonPredicateConverter( + rowType, filterReadsTimestamp(rowType, filter.get()) + && requiresTimestampPrecisionRepair(session, table, paimonHandle)); predicates = converter.convert(filter.get()); } props.put("paimon.predicate", encodeObjectToString(predicates)); // Paimon JDBC metastore options for BE (if applicable) Map backendOptions = new LinkedHashMap<>(getBackendPaimonOptions()); + Set readFieldIds = scanReadFieldIds(table.rowType(), columns, filter); + boolean readsTimestamp = table.rowType().getFields().stream().anyMatch(field -> + readFieldIds.contains(field.id()) + && containsTimestampType(field.type())); + backendOptions.put(DORIS_TIMESTAMP_PRECISION_REPAIR, String.valueOf( + (readsTimestamp || hasTimestampPolicy(table, paimonHandle)) + && requiresTimestampPrecisionRepair(session, table, paimonHandle))); backendManifestCap.ifPresent(cap -> backendOptions.put( DORIS_MANIFEST_PARALLELISM_CAP, String.valueOf(cap))); if (paimonHandle.isSystemTable() && backendManifestCap.isPresent()) { @@ -1337,6 +1359,155 @@ public Map getScanNodeProperties( return props; } + static boolean containsTimestampType(DataType type) { + if (type instanceof org.apache.paimon.types.TimestampType + || type instanceof org.apache.paimon.types.LocalZonedTimestampType) { + return true; + } + if (type instanceof org.apache.paimon.types.RowType) { + return ((org.apache.paimon.types.RowType) type).getFields().stream() + .anyMatch(field -> containsTimestampType(field.type())); + } + if (type instanceof org.apache.paimon.types.ArrayType) { + return containsTimestampType(((org.apache.paimon.types.ArrayType) type).getElementType()); + } + if (type instanceof org.apache.paimon.types.MapType) { + org.apache.paimon.types.MapType mapType = (org.apache.paimon.types.MapType) type; + return containsTimestampType(mapType.getKeyType()) || containsTimestampType(mapType.getValueType()); + } + return false; + } + + static boolean filterReadsTimestamp(RowType rowType, ConnectorExpression filter) { + Set names = new HashSet<>(); + collectFilterColumnNames(filter, names); + return rowType.getFields().stream().anyMatch(field -> + names.contains(field.name().toLowerCase(Locale.ROOT)) && containsTimestampType(field.type())); + } + + private boolean hasTimestampPolicy(Table table, PaimonTableHandle handle) { + // SQL projections may omit timestamp fields required by a secondary branch's catalog policy. + return containsTimestampType(table.rowType()) && withSchemaHistoryAuthentication( + () -> queryAuthEnabled(resolveTimestampHistoryTable(table, handle))); + } + + private static boolean queryAuthEnabled(Table table) { + if (table instanceof FileStoreTable) { + FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase((FileStoreTable) table); + if (base instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) base; + return queryAuthEnabled(pair.wrapped()) || queryAuthEnabled(pair.other()); + } + } + return table != null && new Options(table.options()).get(CoreOptions.QUERY_AUTH_ENABLED); + } + + boolean requiresTimestampPrecisionRepair(ConnectorSession session, Table table, PaimonTableHandle handle) { + if (!containsTimestampType(table.rowType())) { + return false; + } + Supplier lookup = () -> withSchemaHistoryAuthentication( + () -> hasTimestampPrecisionHistory(resolveTimestampHistoryTable(table, handle))); + if (session == null || session.getStatementScope() == null + || session.getStatementScope() == ConnectorStatementScope.NONE) { + return lookup.get(); + } + String memoKey = SCAN_REUSE_NAMESPACE + ":timestamp-history:" + + session.getCatalogId() + ":" + session.getQueryId(); + Map history = session.getStatementScope().computeIfAbsent( + memoKey, () -> new ConcurrentHashMap<>()); + return history.computeIfAbsent(new PaimonTableResolutionKey(handle, effectiveScanOptions(handle)), + ignored -> lookup.get()); + } + + private T withSchemaHistoryAuthentication(Supplier action) { + if (context == null) { + return action.get(); + } + try { + return context.executeAuthenticated(action::get); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to read Paimon timestamp schema history", e); + } + } + + private static boolean hasTimestampPrecisionHistory(Table schemaTable) { + if (!(schemaTable instanceof FileStoreTable)) { + return false; + } + FileStoreTable fileStoreTable = PaimonTableDecorators.unwrapToFallbackOrBase((FileStoreTable) schemaTable); + if (fileStoreTable instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) fileStoreTable; + return hasTimestampPrecisionHistory(pair.wrapped()) || hasTimestampPrecisionHistory(pair.other()); + } + RowType currentType = schemaTable.rowType(); + SchemaManager schemaManager = fileStoreTable.schemaManager(); + for (Long schemaId : schemaManager.listAllIds()) { + TableSchema historicalSchema = schemaManager.schema(schemaId); + if (historicalSchema != null + && hasTimestampPrecisionNarrowing(historicalSchema.logicalRowType(), currentType)) { + return true; + } + } + return false; + } + + static boolean hasTimestampPrecisionNarrowing(DataType historicalType, DataType currentType) { + return hasTimestampPrecisionNarrowing(historicalType, currentType, 0); + } + + private static boolean hasTimestampPrecisionNarrowing( + DataType historicalType, DataType currentType, int minimumHistoricalPrecision) { + return hasTimestampPrecisionNarrowing(historicalType, currentType, minimumHistoricalPrecision, false); + } + + private static boolean hasTimestampPrecisionNarrowing( + DataType historicalType, DataType currentType, int minimumHistoricalPrecision, boolean crossFamilyOnly) { + if ((historicalType instanceof org.apache.paimon.types.TimestampType + || historicalType instanceof org.apache.paimon.types.LocalZonedTimestampType) + && (currentType instanceof org.apache.paimon.types.TimestampType + || currentType instanceof org.apache.paimon.types.LocalZonedTimestampType)) { + // Paimon also casts between timestamp families before truncating precision. + int historicalPrecision = org.apache.paimon.types.DataTypeChecks.getPrecision(historicalType); + int currentPrecision = org.apache.paimon.types.DataTypeChecks.getPrecision(currentType); + return historicalPrecision > currentPrecision && historicalPrecision >= minimumHistoricalPrecision + && (!crossFamilyOnly || historicalType.getTypeRoot() != currentType.getTypeRoot()); + } + if (historicalType instanceof RowType && currentType instanceof RowType) { + RowType historicalRow = (RowType) historicalType; + RowType currentRow = (RowType) currentType; + for (DataField historicalField : historicalRow.getFields()) { + DataField currentField = currentRow.getFields().stream() + .filter(field -> field.id() == historicalField.id()) + .findFirst().orElse(null); + if (currentField != null + && hasTimestampPrecisionNarrowing( + historicalField.type(), currentField.type(), minimumHistoricalPrecision, + crossFamilyOnly)) { + return true; + } + } + return false; + } + if (historicalType instanceof ArrayType && currentType instanceof ArrayType) { + return hasTimestampPrecisionNarrowing( + ((ArrayType) historicalType).getElementType(), ((ArrayType) currentType).getElementType(), + minimumHistoricalPrecision, crossFamilyOnly); + } + if (historicalType instanceof MapType && currentType instanceof MapType) { + MapType historicalMap = (MapType) historicalType; + MapType currentMap = (MapType) currentType; + return hasTimestampPrecisionNarrowing( + historicalMap.getKeyType(), currentMap.getKeyType(), minimumHistoricalPrecision, crossFamilyOnly) + || hasTimestampPrecisionNarrowing( + historicalMap.getValueType(), currentMap.getValueType(), minimumHistoricalPrecision, + crossFamilyOnly); + } + return false; + } + OptionalInt backendManifestParallelism(PaimonTableHandle handle, Table scanTable) { Table planningTable = scanTable; if (handle.isSystemTable()) { @@ -1625,6 +1796,19 @@ private static FileStoreTable rebuildWithoutCatalogLoader(FileStoreTable branch) CatalogEnvironment.empty()).copy(branch.schema()); } + private Table resolveTimestampHistoryTable(Table table, PaimonTableHandle handle) { + if (handle.isSystemTable() && Arrays.asList("audit_log", "binlog", "row_tracking") + .contains(handle.getSysTableName())) { + Table pinned = handle.getSystemTableSource(); + if (pinned == null) { + pinned = handle.getSysBaseTable(); + } + return pinned == null ? prepareSystemSource(handle, reloadBaseTable(handle)) + : prepareSystemSource(handle, pinned); + } + return resolveSchemaDictTable(table, handle); + } + /** * Resolves the {@link FileStoreTable} whose schema dictionary BE needs to field-id-match the native * data files for {@code table}. A normal data table IS the FileStoreTable. A read-optimized system @@ -2076,6 +2260,17 @@ private static void collectFilterColumnNames(ConnectorExpression expression, Set expression.getChildren().forEach(child -> collectFilterColumnNames(child, names)); } + private static Set branchReadFieldIds(RowType scanType, RowType branchType, Set readFieldIds) { + // Fallback schemas agree positionally, but field IDs and names may evolve independently. + Set branchIds = new HashSet<>(); + for (int i = 0; i < scanType.getFieldCount(); i++) { + if (readFieldIds.contains(scanType.getFields().get(i).id())) { + branchIds.add(branchType.getFields().get(i).id()); + } + } + return branchIds; + } + static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) { FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase(table); if (base instanceof FallbackReadFileStoreTable) { @@ -2087,27 +2282,61 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) return base; } - static boolean requiresLegacyOrcTimestampReader(Table table, Optional> rawFiles, + private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional> rawFiles, + Set readFieldIds, Map schemaTimestamps) { + RowType readType = new RowType(table.rowType().getFields().stream() + .filter(field -> readFieldIds.contains(field.id())).collect(Collectors.toList())); + if (!containsTimestampType(readType)) { + return false; + } + for (RawFile file : rawFiles.get()) { + if (schemaTimestamps.computeIfAbsent(file.schemaId(), id -> hasTimestampPrecisionNarrowing( + table.schemaManager().schema(id).logicalRowType(), readType, 0, true))) { + // Generic native casts round after zone conversion. The SDK keeps its conversion + // semantics while the JNI widened read type delays truncation until materialization. + return true; + } + } + return false; + } + + static boolean requiresOrcTimestampReader(Table table, Optional> rawFiles, Set readFieldIds, Map schemaTimestamps) { if (readFieldIds.isEmpty() || !rawFiles.isPresent() - || rawFiles.get().stream().noneMatch(f -> f.path().endsWith(".orc")) - || !new org.apache.paimon.options.Options(table.options()).get( - org.apache.paimon.format.OrcOptions.ORC_TIMESTAMP_LTZ_LEGACY_TYPE)) { + || rawFiles.get().stream().noneMatch(f -> f.path().endsWith(".orc"))) { return false; } // Only fields present in a file require SDK timezone conversion. A newly added nullable LTZ // column reads as NULL in older files and must not disable their native metadata scans. FileStoreTable fileStoreTable = (FileStoreTable) table; + boolean legacyLtz = new org.apache.paimon.options.Options(table.options()).get( + org.apache.paimon.format.OrcOptions.ORC_TIMESTAMP_LTZ_LEGACY_TYPE); + RowType readType = new RowType(fileStoreTable.rowType().getFields().stream() + .filter(field -> readFieldIds.contains(field.id())).collect(Collectors.toList())); + if (!legacyLtz && !containsTimestampType(readType)) { + return false; + } for (RawFile file : rawFiles.get()) { if (file.path().endsWith(".orc") && schemaTimestamps.computeIfAbsent(file.schemaId(), - id -> readsTimestampLtz( - fileStoreTable.schemaManager().schema(id).logicalRowType(), readFieldIds))) { + id -> { + RowType historicalType = fileStoreTable.schemaManager().schema(id).logicalRowType(); + // Native ORC rounds nanoseconds before the precision cast; JNI keeps the + // original fraction so a historical p7-p9 value cannot cross a second boundary. + return (legacyLtz && readsTimestampLtz(historicalType, readFieldIds)) + || hasTimestampPrecisionNarrowing(historicalType, readType, 7); + })) { return true; } } return false; } + // Keep the package-visible helper name used by tests added on the target branch. + static boolean requiresLegacyOrcTimestampReader(Table table, Optional> rawFiles, + Set readFieldIds, Map schemaTimestamps) { + return requiresOrcTimestampReader(table, rawFiles, readFieldIds, schemaTimestamps); + } + private static boolean readsTimestampLtz(RowType rowType, Set readFieldIds) { return rowType.getFields().stream().anyMatch( field -> readFieldIds.contains(field.id()) && containsTimestampLtz(field.type())); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLegacyOrcTimestampTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLegacyOrcTimestampTest.java index 7d0221494b804f..0fbd2867762eb9 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLegacyOrcTimestampTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLegacyOrcTimestampTest.java @@ -57,22 +57,22 @@ void routeCurrentAndHistoricalNestedLtzToSdk(@TempDir Path warehouse) throws Exc FileStoreTable original = (FileStoreTable) catalog.getTable(id); Optional> oldFiles = Optional.of(Collections.singletonList( new RawFile("part.orc", 0L, 100L, 1L, "orc", original.schema().id(), 0L))); - Assertions.assertTrue(PaimonScanPlanProvider.requiresLegacyOrcTimestampReader( + Assertions.assertTrue(PaimonScanPlanProvider.requiresOrcTimestampReader( original, oldFiles, Collections.singleton(1), new HashMap<>())); new SchemaManager(original.fileIO(), original.location()) .commitChanges(SchemaChange.dropColumn("events")); FileStoreTable evolved = (FileStoreTable) catalog.getTable(id); Map schemaCache = new HashMap<>(); // Historical field IDs, rather than all file columns, determine whether LTZ is decoded. - Assertions.assertFalse(PaimonScanPlanProvider.requiresLegacyOrcTimestampReader( + Assertions.assertFalse(PaimonScanPlanProvider.requiresOrcTimestampReader( evolved, oldFiles, Collections.singleton(0), new HashMap<>())); - Assertions.assertTrue(PaimonScanPlanProvider.requiresLegacyOrcTimestampReader( + Assertions.assertTrue(PaimonScanPlanProvider.requiresOrcTimestampReader( evolved, oldFiles, Collections.singleton(1), schemaCache)); Assertions.assertEquals(Collections.singletonMap(original.schema().id(), true), schemaCache); - Assertions.assertFalse(PaimonScanPlanProvider.requiresLegacyOrcTimestampReader( + Assertions.assertFalse(PaimonScanPlanProvider.requiresOrcTimestampReader( evolved.copy(Collections.singletonMap(OrcOptions.ORC_TIMESTAMP_LTZ_LEGACY_TYPE.key(), "false")), oldFiles, Collections.singleton(1), new HashMap<>())); - Assertions.assertFalse(PaimonScanPlanProvider.requiresLegacyOrcTimestampReader(evolved, + Assertions.assertFalse(PaimonScanPlanProvider.requiresOrcTimestampReader(evolved, Optional.of(Collections.singletonList( new RawFile("part.parquet", 0L, 100L, 1L, "parquet", original.schema().id(), 0L))), Collections.singleton(1), new HashMap<>())); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPredicateConverterTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPredicateConverterTest.java index 65a8af2be21a15..10faf8a23002a6 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPredicateConverterTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonPredicateConverterTest.java @@ -18,12 +18,14 @@ package org.apache.doris.connector.paimon; import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorAnd; import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; import org.apache.doris.connector.spi.pushdown.ConnectorComparison; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; import org.apache.doris.connector.spi.pushdown.ConnectorIn; import org.apache.doris.connector.spi.pushdown.ConnectorLike; import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.pushdown.ConnectorOr; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; @@ -104,6 +106,49 @@ public void ntzPushPreservesMicroseconds() { "an NTZ predicate literal must retain precision below one millisecond"); } + @Test + public void precisionRepairDropsOnlyTimestampPredicates() { + RowType rowType = RowType.builder() + .field("id", DataTypes.INT()) + .field("ts", DataTypes.TIMESTAMP(6)) + .build(); + PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType, true); + + ConnectorComparison idComparison = new ConnectorComparison( + ConnectorComparison.Operator.EQ, new ConnectorColumnRef("id", ANY), + new ConnectorLiteral(ANY, 7)); + ConnectorComparison timestampComparison = new ConnectorComparison( + ConnectorComparison.Operator.EQ, new ConnectorColumnRef("ts", ANY), + new ConnectorLiteral(ANY, LocalDateTime.of(2025, 1, 1, 0, 0))); + + Assertions.assertEquals(1, converter.convert(idComparison).size(), + "non-timestamp predicates must remain pushable during precision repair"); + Assertions.assertTrue(converter.convert(timestampComparison).isEmpty(), + "timestamp predicates must remain Doris residuals during precision repair"); + } + + @Test + public void precisionRepairKeepsSafeAndSiblingsButNotMixedOr() { + RowType rowType = RowType.builder() + .field("id", DataTypes.INT()) + .field("ts", DataTypes.TIMESTAMP(6)) + .build(); + PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType, true); + ConnectorComparison idComparison = new ConnectorComparison( + ConnectorComparison.Operator.EQ, new ConnectorColumnRef("id", ANY), + new ConnectorLiteral(ANY, 7)); + ConnectorComparison timestampComparison = new ConnectorComparison( + ConnectorComparison.Operator.EQ, new ConnectorColumnRef("ts", ANY), + new ConnectorLiteral(ANY, LocalDateTime.of(2025, 1, 1, 0, 0))); + + Assertions.assertEquals(1, converter.convert(new ConnectorAnd( + Arrays.asList(idComparison, timestampComparison))).size(), + "a safe AND sibling must remain available for file pruning"); + Assertions.assertTrue(converter.convert(new ConnectorOr( + Arrays.asList(idComparison, timestampComparison))).isEmpty(), + "a mixed OR must remain a residual instead of narrowing the result set"); + } + @Test public void nanosecondTimestampComparisonsRemainResidual() { RowType rowType = RowType.builder().field("ts", DataTypes.TIMESTAMP(9)).build(); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 81570d6f1b9375..eab9feca0e463f 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -138,6 +138,95 @@ private static RowType rowType(String... columnNames) { return builder.build(); } + @Test + public void timestampPredicatesAreMarkedUnsafeForPaimonPlanning() { + RowType nested = RowType.builder() + .field("payload", DataTypes.ROW(DataTypes.TIMESTAMP(4))) + .field("id", DataTypes.INT()) + .build(); + RowType plain = RowType.builder().field("id", DataTypes.INT()).build(); + + Assertions.assertTrue(PaimonScanPlanProvider.containsTimestampType(nested)); + Assertions.assertFalse(PaimonScanPlanProvider.containsTimestampType(plain)); + } + + @Test + public void timestampPredicateRepairOnlyAppliesToNarrowedHistoricalSchemas() { + DataType current = DataTypes.ROW( + DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(0)), + DataTypes.FIELD(1, "id", DataTypes.INT())); + DataType samePrecision = DataTypes.ROW( + DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(0)), + DataTypes.FIELD(1, "id", DataTypes.INT())); + DataType widenedHistorical = DataTypes.ROW( + DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(6)), + DataTypes.FIELD(1, "id", DataTypes.INT())); + + Assertions.assertFalse(PaimonScanPlanProvider.hasTimestampPrecisionNarrowing( + samePrecision, current)); + Assertions.assertTrue(PaimonScanPlanProvider.hasTimestampPrecisionNarrowing( + widenedHistorical, current)); + } + + @Test + public void crossFamilyTimestampHistoryEnablesRepair(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + for (boolean toLtz : new boolean[] {false, true}) { + DataType historical = toLtz ? DataTypes.TIMESTAMP(6) + : new org.apache.paimon.types.LocalZonedTimestampType(6); + DataType current = toLtz ? new org.apache.paimon.types.LocalZonedTimestampType(0) + : DataTypes.TIMESTAMP(0); + Assertions.assertTrue(PaimonScanPlanProvider.hasTimestampPrecisionNarrowing(historical, current)); + Assertions.assertFalse(PaimonScanPlanProvider.hasTimestampPrecisionNarrowing(current, historical)); + DataType equalPrecision = toLtz ? new org.apache.paimon.types.LocalZonedTimestampType(6) + : DataTypes.TIMESTAMP(6); + Assertions.assertFalse(PaimonScanPlanProvider.hasTimestampPrecisionNarrowing( + historical, equalPrecision)); + Identifier id = Identifier.create("db", "cross_family_" + toLtz); + catalog.createTable(id, Schema.newBuilder().column("ts", historical) + .option("file.format", "parquet").build(), false); + Table table = catalog.getTable(id); + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + write.write(GenericRow.of(org.apache.paimon.data.Timestamp.fromLocalDateTime( + java.time.LocalDateTime.of(1969, 12, 31, 23, 59, 59, 600_000_000)))); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(id, Collections.singletonList( + SchemaChange.updateColumnType("ts", current, true)), false); + table = catalog.getTable(id); + PaimonTableHandle handle = new PaimonTableHandle( + "db", id.getObjectName(), Collections.emptyList(), Collections.emptyList()); + handle.setPaimonTable(table); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + ConnectorSession session = sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")); + List columns = Collections.singletonList(new PaimonColumnHandle("ts", 0)); + ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of(toLtz ? "TIMESTAMPTZ" : "DATETIMEV2")), + ConnectorLiteral.ofDatetime(java.time.LocalDateTime.of(1969, 12, 31, 23, 59, 59))); + Assertions.assertFalse(provider.planScan(session, + ConnectorScanRequest.builder(handle, columns).filter(Optional.of(filter)).build()).isEmpty()); + Map props = provider.getScanNodeProperties(session, handle, columns, Optional.of(filter)); + Assertions.assertTrue(props.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"true\"")); + List predicates = InstantiationUtil.deserializeObject( + Base64.getDecoder().decode(props.get("paimon.predicate")), getClass().getClassLoader()); + Assertions.assertTrue(predicates.isEmpty(), "cross-family timestamp equality must remain residual"); + Assertions.assertTrue(provider.planScan(sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handle, columns).build()).stream() + .allMatch(range -> !range.getPath().isPresent()), + "native cross-family narrowing must use the SDK instead of a rounding SQL cast"); + } + } + } + private static final class OrderedLocalFileIO extends LocalFileIO { @Override public FileStatus[] listFiles(org.apache.paimon.fs.Path path, boolean recursive) @@ -150,6 +239,263 @@ public FileStatus[] listFiles(org.apache.paimon.fs.Path path, boolean recursive) } } + @Test + public void timestampHistoryChecksFallbackBranchesAndReusesStatementResult(@TempDir Path warehouse) + throws Exception { + RecordingConnectorContext context = new RecordingConnectorContext(); + TimestampHistoryFileIO fileIO = new TimestampHistoryFileIO(context); + try (Catalog catalog = new FileSystemCatalog(fileIO, new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier mainId = Identifier.create("db", "main"); + Identifier fallbackId = Identifier.create("db", "fallback"); + catalog.createTable(mainId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(4)).partitionKeys("id").build(), false); + catalog.createTable(fallbackId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(6)).partitionKeys("id") + .option("row-tracking.enabled", "true").build(), false); + FileStoreTable fallback = (FileStoreTable) catalog.getTable(fallbackId); + BatchWriteBuilder writer = fallback.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + write.write(GenericRow.of(7, org.apache.paimon.data.Timestamp.fromLocalDateTime( + java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1, 1000)))); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(fallbackId, Collections.singletonList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(4), true)), false); + FileStoreTable main = (FileStoreTable) catalog.getTable(mainId); + fallback = (FileStoreTable) catalog.getTable(fallbackId); + Table pair = new FallbackReadFileStoreTable(main, fallback, true); + PaimonTableHandle handle = new PaimonTableHandle( + "db", "main", Collections.singletonList("id"), Collections.emptyList()); + handle.setPaimonTable(pair); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = pair; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops, context); + ConnectorSession session = sessionWithProps(Collections.singletonMap("force_jni_scanner", "true"), + new TestStatementScope()); + ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("DATETIMEV2")), + ConnectorLiteral.ofDatetime(java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1))); + fileIO.requireAuthentication = true; + Assertions.assertTrue(provider.requiresTimestampPrecisionRepair(session, pair, handle)); + int lists = fileIO.schemaLists; + // Each SPI path may use a fresh provider; the statement scope must own the memo. + provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, context); + List columns = Collections.singletonList(new PaimonColumnHandle("ts", 1)); + Assertions.assertFalse(provider.planScan(session, + ConnectorScanRequest.builder(handle, columns).filter(Optional.of(filter)).build()).isEmpty(), + "fallback-only .000001 must survive FE equality pruning"); + Map props = provider.getScanNodeProperties(session, handle, columns, Optional.of(filter)); + List predicates = InstantiationUtil.deserializeObject( + Base64.getDecoder().decode(props.get("paimon.predicate")), getClass().getClassLoader()); + Assertions.assertTrue(predicates.isEmpty()); + Assertions.assertTrue(props.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"true\"")); + Assertions.assertEquals(lists, fileIO.schemaLists); + Assertions.assertTrue(provider.requiresTimestampPrecisionRepair( + sessionWithProps(Collections.emptyMap()), new FallbackReadFileStoreTable(fallback, main, true), + handle), "the narrowed branch can also be the wrapped branch"); + PaimonTableHandle ro = PaimonTableHandle.forSystemTable("db", "fallback", "ro", false); + ro.setSysBaseTable(fallback); + Table readOptimized = new ReadOptimizedTable(fallback); + ro.setPaimonTable(readOptimized); + Assertions.assertTrue(provider.requiresTimestampPrecisionRepair( + sessionWithProps(Collections.emptyMap()), readOptimized, ro), + "$ro history reads must also stay inside the plugin authentication scope"); + for (String kind : Arrays.asList("audit_log", "binlog", "row_tracking")) { + PaimonTableHandle system = PaimonTableHandle.forSystemTable("db", "fallback", kind, true); + system.setSysBaseTable(fallback); + system.setSystemTableSource(fallback); + Table wrapper = org.apache.paimon.table.system.SystemTableLoader.load(kind, fallback); + system.setPaimonTable(wrapper); + Assertions.assertTrue(provider.requiresTimestampPrecisionRepair( + sessionWithProps(Collections.emptyMap()), wrapper, system), kind); + } + FileStoreTable authorized = fallback.copy(Collections.singletonMap("query-auth.enabled", "true")); + PaimonTableHandle authHandle = new PaimonTableHandle( + "db", "fallback", Collections.singletonList("id"), Collections.emptyList()); + authHandle.setPaimonTable(authorized); + Map authProps = provider.getScanNodeProperties( + sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")), authHandle, + Collections.singletonList(new PaimonColumnHandle("id", 0)), Optional.empty()); + Assertions.assertTrue(authProps.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"true\""), + "a catalog policy can read timestamps absent from the SQL projection"); + for (boolean wrappedFirst : new boolean[] {true, false}) { + FileStoreTable authPair = wrappedFirst + ? new FallbackReadFileStoreTable(main, authorized, true) + : new FallbackReadFileStoreTable(main.copy(Collections.singletonMap( + org.apache.paimon.CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")), authorized, false); + for (String kind : Arrays.asList("", "audit_log")) { + Table scanTable = kind.isEmpty() ? authPair + : org.apache.paimon.table.system.SystemTableLoader.load(kind, authPair); + PaimonTableHandle policyHandle = kind.isEmpty() ? new PaimonTableHandle( + "db", "main", Collections.singletonList("id"), Collections.emptyList()) + : PaimonTableHandle.forSystemTable("db", "main", kind, true); + policyHandle.setPaimonTable(scanTable); + if (!kind.isEmpty()) { + policyHandle.setSysBaseTable(authPair); + policyHandle.setSystemTableSource(authPair); + } + Map policyProps = provider.getScanNodeProperties( + sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")), policyHandle, + Collections.singletonList(new PaimonColumnHandle("id", 0)), Optional.empty()); + Assertions.assertTrue(policyProps.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"true\""), + "hidden timestamp policies on the secondary branch require repair: " + kind); + } + } + context.failAuth = true; + Assertions.assertThrows(RuntimeException.class, () -> new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops, context) + .requiresTimestampPrecisionRepair(sessionWithProps(Collections.emptyMap()), pair, handle)); + } + } + + @Test + public void unrelatedPredicatesSkipTimestampHistory(@TempDir Path warehouse) throws Exception { + RecordingConnectorContext context = new RecordingConnectorContext(); + TimestampHistoryFileIO fileIO = new TimestampHistoryFileIO(context); + try (Catalog catalog = new FileSystemCatalog(fileIO, new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + for (boolean hasTimestamp : new boolean[] {false, true}) { + Identifier id = Identifier.create("db", "table_" + hasTimestamp); + Schema.Builder builder = Schema.newBuilder().column("id", DataTypes.INT()); + if (hasTimestamp) { + builder.column("ts", DataTypes.TIMESTAMP(4)); + } + catalog.createTable(id, builder.build(), false); + Table table = catalog.getTable(id); + PaimonTableHandle handle = new PaimonTableHandle( + "db", id.getObjectName(), Collections.emptyList(), Collections.emptyList()); + handle.setPaimonTable(table); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops, context); + ConnectorSession session = sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")); + ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("id", ConnectorType.of("INT")), ConnectorLiteral.ofInt(1)); + int lists = fileIO.schemaLists; + fileIO.requireAuthentication = true; + List columns = Collections.singletonList(new PaimonColumnHandle("id", 0)); + provider.planScan(session, ConnectorScanRequest.builder(handle, columns) + .filter(Optional.of(filter)).build()); + Map props = provider.getScanNodeProperties( + session, handle, columns, Optional.of(filter)); + Assertions.assertEquals(lists, fileIO.schemaLists); + List predicates = InstantiationUtil.deserializeObject( + Base64.getDecoder().decode(props.get("paimon.predicate")), getClass().getClassLoader()); + Assertions.assertEquals(1, predicates.size()); + Assertions.assertTrue(props.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"false\"")); + handle.setPaimonTable(table.copy(Collections.singletonMap("query-auth.enabled", "true"))); + Map policyProps = provider.getScanNodeProperties( + session, handle, columns, Optional.of(filter)); + Assertions.assertTrue(policyProps.get("paimon.options_json").contains( + "\"doris.timestamp-precision-repair\":\"false\""), + "query auth alone must not enable repair without historical narrowing"); + if (!hasTimestamp) { + Assertions.assertEquals(lists, fileIO.schemaLists); + } + fileIO.requireAuthentication = false; + } + } + } + + private static final class TimestampHistoryFileIO extends LocalFileIO { + private final transient RecordingConnectorContext context; + private boolean requireAuthentication; + private int schemaLists; + + private TimestampHistoryFileIO(RecordingConnectorContext context) { + this.context = context; + } + + @Override + public FileStatus[] listFiles(org.apache.paimon.fs.Path path, boolean recursive) throws IOException { + if (path.getName().equals("schema")) { + schemaLists++; + if (requireAuthentication) { + Assertions.assertTrue(context.authDepth > 0, "schema listing must be authenticated"); + } + } + return super.listFiles(path, recursive); + } + + @Override + public org.apache.paimon.fs.SeekableInputStream newInputStream(org.apache.paimon.fs.Path path) + throws IOException { + if (requireAuthentication && path.getName().startsWith("schema-")) { + Assertions.assertTrue(context.authDepth > 0, "schema loading must be authenticated"); + } + return super.newInputStream(path); + } + } + + @Test + public void narrowedNanosecondOrcUsesJniOnlyForAffectedFields(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + for (int precision : new int[] {6, 7, 9}) { + Identifier id = Identifier.create("db", "orc_" + precision); + catalog.createTable(id, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(precision)) + .column("ltz", new org.apache.paimon.types.LocalZonedTimestampType(precision)) + .option("file.format", "orc") + .option(org.apache.paimon.format.OrcOptions.ORC_TIMESTAMP_LTZ_LEGACY_TYPE.key(), "false") + .build(), false); + Table original = catalog.getTable(id); + BatchWriteBuilder writer = original.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + org.apache.paimon.data.Timestamp value = org.apache.paimon.data.Timestamp.fromLocalDateTime( + java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1, 999_999_500)); + write.write(GenericRow.of(1, value, value)); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(id, Arrays.asList( + SchemaChange.updateColumnType("ts", DataTypes.TIMESTAMP(0), true), + SchemaChange.updateColumnType("ltz", new org.apache.paimon.types.LocalZonedTimestampType(0), + true)), false); + Table evolved = catalog.getTable(id); + PaimonTableHandle handle = new PaimonTableHandle( + "db", id.getObjectName(), Collections.emptyList(), Collections.emptyList()); + handle.setPaimonTable(evolved); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = evolved; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + ConnectorSession session = sessionWithProps(Collections.emptyMap()); + for (String name : Arrays.asList("ts", "ltz")) { + List ranges = provider.planScan(session, ConnectorScanRequest.builder( + handle, Collections.singletonList(new PaimonColumnHandle(name, name.equals("ts") ? 1 : 2))) + .build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertEquals(precision <= 6, ranges.stream().allMatch(r -> r.getPath().isPresent()), + "only p7-p9 narrowing needs the SDK nanosecond reader"); + } + Assertions.assertTrue(provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new PaimonColumnHandle("id", 0))).build()) + .stream().allMatch(r -> r.getPath().isPresent())); + ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("DATETIMEV2")), + ConnectorLiteral.ofDatetime(java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1))); + List ranges = provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new PaimonColumnHandle("id", 0))) + .filter(Optional.of(filter)).build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertEquals(precision <= 6, ranges.stream().allMatch(r -> r.getPath().isPresent()), + "filter-only timestamps need the same routing as projected timestamps"); + } + } + } + @Test public void resolveTableReloadsWhenTransientTableNull() { RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); @@ -3100,6 +3446,77 @@ public void addedLtzColumnKeepsHistoricalOrcFilesOnNativeReader(@TempDir Path wa } } + @Test + public void fallbackTimestampRoutingTranslatesIndependentFieldIds(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + for (boolean crossFamily : new boolean[] {false, true}) { + Identifier mainId = Identifier.create("db", "main_" + crossFamily); + Identifier otherId = Identifier.create("db", "other_" + crossFamily); + org.apache.paimon.types.DataType current = DataTypes.TIMESTAMP(0); + org.apache.paimon.types.DataType historical = crossFamily + ? new org.apache.paimon.types.LocalZonedTimestampType(6) : DataTypes.TIMESTAMP(9); + catalog.createTable(mainId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", current).partitionKeys("id").build(), false); + catalog.alterTable(mainId, Collections.singletonList(SchemaChange.dropColumn("ts")), false); + catalog.alterTable(mainId, Collections.singletonList(SchemaChange.addColumn("ts", current)), false); + catalog.createTable(otherId, Schema.newBuilder().column("id", DataTypes.INT()) + .column("ts", historical).partitionKeys("id") + .option("file.format", crossFamily ? "parquet" : "orc").build(), false); + // Align schema IDs while leaving the two branches' field IDs independent. + catalog.alterTable(otherId, Collections.singletonList(SchemaChange.setOption("test.option", "1")), + false); + catalog.alterTable(otherId, Collections.singletonList(SchemaChange.setOption("test.option", "2")), + false); + FileStoreTable other = (FileStoreTable) catalog.getTable(otherId); + BatchWriteBuilder writer = other.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + write.write(GenericRow.of(7, org.apache.paimon.data.Timestamp.fromLocalDateTime( + java.time.LocalDateTime.of(2025, 1, 1, 0, 0, 1, 999999500)))); + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + catalog.alterTable(otherId, Arrays.asList( + SchemaChange.updateColumnType("ts", current, true), + SchemaChange.renameColumn("ts", "branch_ts")), false); + FileStoreTable main = (FileStoreTable) catalog.getTable(mainId); + other = (FileStoreTable) catalog.getTable(otherId); + Assertions.assertNotEquals(main.rowType().getFields().get(1).id(), + other.rowType().getFields().get(1).id()); + for (boolean wrappedFirst : new boolean[] {true, false}) { + FileStoreTable wrapped = wrappedFirst ? main : main.copy(Collections.singletonMap( + org.apache.paimon.CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")); + Table pair = new FallbackReadFileStoreTable(wrapped, other, wrappedFirst); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = pair; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle( + "db", mainId.getObjectName(), Collections.singletonList("id"), Collections.emptyList()); + ConnectorSession session = sessionWithProps(Collections.emptyMap()); + List ranges = provider.planScan(session, + ConnectorScanRequest.builder(handle, + Collections.singletonList(new PaimonColumnHandle("ts", 1))).build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertTrue(ranges.stream().noneMatch(range -> range.getPath().isPresent()), + "historical timestamp routing must use the selected branch field IDs"); + ranges = provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new PaimonColumnHandle("id", 0))) + .filter(Optional.of(new org.apache.doris.connector.spi.pushdown.ConnectorIsNull( + new ConnectorColumnRef("ts", ConnectorType.of("DATETIMEV2")), true))).build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertTrue(ranges.stream().noneMatch(range -> range.getPath().isPresent())); + ranges = provider.planScan(session, ConnectorScanRequest.builder(handle, + Collections.singletonList(new PaimonColumnHandle("id", 0))).build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertTrue(ranges.stream().allMatch(range -> range.getPath().isPresent())); + } + } + } + } + @Test public void fallbackOrcUsesEachBranchOptionsAndSchemas(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -3120,6 +3537,10 @@ public void fallbackOrcUsesEachBranchOptionsAndSchemas(@TempDir Path warehouse) catalog.alterTable(id, Collections.singletonList(SchemaChange.addColumn("unused", DataTypes.INT())), false); catalog.alterTable(id, Collections.singletonList(SchemaChange.dropColumn("unused")), false); + } else { + catalog.alterTable(id, Collections.singletonList(SchemaChange.dropColumn("event_time")), false); + catalog.alterTable(id, Collections.singletonList(SchemaChange.addColumn("event_time", + new org.apache.paimon.types.LocalZonedTimestampType(6))), false); } FileStoreTable branchTable = (FileStoreTable) catalog.getTable(id); BatchWriteBuilder writer = branchTable.newBatchWriteBuilder(); diff --git a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out index ec7e602af4b950..f7f6b369ee224d 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out +++ b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out @@ -1,4 +1,143 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !nanosecond_evolution_v2 -- +1 2025-01-01 00:00:01 2025-01-01 08:00:01+08:00 0 0 +2 1969-12-31 23:59:59 2025-01-01 08:00:01+08:00 0 0 + +-- !nanosecond_evolution_v2_predicate -- +1 + +-- !nanosecond_evolution_ltz_predicate -- +1 +2 + +-- !nanosecond_evolution_v1 -- +1 2025-01-01 00:00:01 2025-01-01 08:00:01+08:00 0 0 +2 1969-12-31 23:59:59 2025-01-01 08:00:01+08:00 0 0 + +-- !nanosecond_evolution_v1_predicate -- +1 + +-- !nanosecond_evolution_jni -- +1 2025-01-01 00:00:01 2025-01-01 08:00:01+08:00 0 0 +2 1969-12-31 23:59:59 2025-01-01 08:00:01+08:00 0 0 + +-- !nanosecond_evolution_jni_predicate -- +1 + +-- !precision_evolution_v2_parquet -- +1 2025-01-01 00:00:01 0 +2 1969-12-31 23:59:59 0 +3 2025-01-01 00:00:03 0 + +-- !precision_evolution_v2_orc -- +1 2025-01-01 00:00:01 0 +2 1969-12-31 23:59:59 0 +3 2025-01-01 00:00:03 0 + +-- !precision_evolution_v2_parquet_predicate -- +1 2025-01-01 00:00:01 0 + +-- !precision_evolution_v2_orc_predicate -- +1 2025-01-01 00:00:01 0 + +-- !precision_evolution_jni_parquet -- +1 2025-01-01 00:00:01 0 +2 1969-12-31 23:59:59 0 +3 2025-01-01 00:00:03 0 + +-- !precision_evolution_jni_orc -- +1 2025-01-01 00:00:01 0 +2 1969-12-31 23:59:59 0 +3 2025-01-01 00:00:03 0 + +-- !precision_evolution_jni_parquet_predicate -- +1 2025-01-01 00:00:01 0 + +-- !precision_evolution_jni_orc_predicate -- +1 2025-01-01 00:00:01 0 + +-- !leading_zero_p4_jni_parquet -- +1 2025-01-01 00:00:01.0000 0 + +-- !leading_zero_p4_jni_orc -- +1 2025-01-01 00:00:01.0000 0 + +-- !timestamp_map_key_jni_reader -- +1 {"2025-01-01 00:00:01.000001":"1969-12-31 23:59:59.000001"} + +-- !timestamp_map_key_jni_lookup -- +1969-12-31T23:59:59.000001 + +-- !timestamp_nested_jni_projection -- +["1969-12-31 23:59:59.0000", "1970-01-01 00:00:00.1234"] {"nested_ts":"1969-12-31 23:59:59.0000"} + +-- !timestamp_nested_jni_predicate -- +1969-12-31T23:59:59 1969-12-31T23:59:59 + +-- !cross_family_UTC_projection -- +1 true true 0 0 +2 true true 0 0 + +-- !cross_family_UTC_ntz_to_ltz_eq_1 -- +1 + +-- !cross_family_UTC_ntz_to_ltz_eq_2 -- +2 + +-- !cross_family_UTC_ltz_to_ntz_eq_1 -- +1 + +-- !cross_family_UTC_ltz_to_ntz_eq_2 -- +2 + +-- !cross_family_UTC_automatic_projection -- +1 true true 0 0 +2 true true 0 0 + +-- !cross_family_UTC_automatic_ntz_to_ltz_eq_1 -- +1 + +-- !cross_family_UTC_automatic_ntz_to_ltz_eq_2 -- +2 + +-- !cross_family_UTC_automatic_ltz_to_ntz_eq_1 -- +1 + +-- !cross_family_UTC_automatic_ltz_to_ntz_eq_2 -- +2 + +-- !cross_family_Asia_Shanghai_projection -- +1 true true 0 0 +2 true true 0 0 + +-- !cross_family_Asia_Shanghai_ntz_to_ltz_eq_1 -- +1 + +-- !cross_family_Asia_Shanghai_ntz_to_ltz_eq_2 -- +2 + +-- !cross_family_Asia_Shanghai_ltz_to_ntz_eq_1 -- +1 + +-- !cross_family_Asia_Shanghai_ltz_to_ntz_eq_2 -- +2 + +-- !cross_family_Asia_Shanghai_automatic_projection -- +1 true true 0 0 +2 true true 0 0 + +-- !cross_family_Asia_Shanghai_automatic_ntz_to_ltz_eq_1 -- +1 + +-- !cross_family_Asia_Shanghai_automatic_ntz_to_ltz_eq_2 -- +2 + +-- !cross_family_Asia_Shanghai_automatic_ltz_to_ntz_eq_1 -- +1 + +-- !cross_family_Asia_Shanghai_automatic_ltz_to_ntz_eq_2 -- +2 + -- !c1 -- 1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 diff --git a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy index b3e3f35873a85d..6a003d13d39b53 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy @@ -24,8 +24,9 @@ suite("paimon_timestamp_types", "p0,external") { } def originalTimeZone = sql("SELECT @@time_zone")[0][0] - def originalScannerV2 = sql("SHOW VARIABLES LIKE 'enable_file_scanner_v2'")[0][1] - def originalForceJni = sql("SHOW VARIABLES LIKE 'force_jni_scanner'")[0][1] + def originalSettings = ["enable_file_scanner_v2", "force_jni_scanner"].collectEntries { name -> + [(name): sql("show variables like '${name}'")[0][1]] + } try { String catalog_name = "paimon_timestamp_types" String minio_port = context.config.otherConfigs.get("iceberg_minio_port") @@ -145,6 +146,146 @@ suite("paimon_timestamp_types", "p0,external") { } + def precisionEvolutionQuery = { table -> + return """ + select id, cast(ts as string), microsecond(ts) + from ${table} + order by id + """ + } + + def precisionEvolutionPredicateQuery = { table -> + return """ + select id, cast(ts as string), microsecond(ts) + from ${table} + where ts = '2025-01-01 00:00:01' + order by id + """ + } + + sql """set enable_file_scanner_v2=true""" + // Historical ORC p9 fractions must be truncated before native microsecond rounding. + // The FE routes only these affected files to JNI, including filter-only timestamps. + def nanosecondEvolutionQuery = """ + select id, cast(ts as string), cast(ltz as string), microsecond(ts), microsecond(ltz) + from timestamp_nanosecond_evolution_orc order by id + """ + def nanosecondEvolutionPredicate = """ + select id from timestamp_nanosecond_evolution_orc + where ts = '2025-01-01 00:00:01' order by id + """ + sql """set force_jni_scanner=false""" + order_qt_nanosecond_evolution_v2 nanosecondEvolutionQuery + order_qt_nanosecond_evolution_v2_predicate nanosecondEvolutionPredicate + order_qt_nanosecond_evolution_ltz_predicate """ + select id from timestamp_nanosecond_evolution_orc + where ltz = '2025-01-01 08:00:01' order by id + """ + sql """set enable_file_scanner_v2=false""" + order_qt_nanosecond_evolution_v1 nanosecondEvolutionQuery + order_qt_nanosecond_evolution_v1_predicate nanosecondEvolutionPredicate + sql """set force_jni_scanner=true""" + order_qt_nanosecond_evolution_jni nanosecondEvolutionQuery + order_qt_nanosecond_evolution_jni_predicate nanosecondEvolutionPredicate + sql """set force_jni_scanner=false""" + sql """set enable_file_scanner_v2=true""" + order_qt_precision_evolution_v2_parquet precisionEvolutionQuery( + "timestamp_precision_evolution_parquet") + order_qt_precision_evolution_v2_orc precisionEvolutionQuery( + "timestamp_precision_evolution_orc") + // These fixtures evolve from TIMESTAMP(6) to TIMESTAMP(0). Keep a native-reader + // predicate case to verify both filtering and projection use the SDK-truncated value. + order_qt_precision_evolution_v2_parquet_predicate precisionEvolutionPredicateQuery( + "timestamp_precision_evolution_parquet") + order_qt_precision_evolution_v2_orc_predicate precisionEvolutionPredicateQuery( + "timestamp_precision_evolution_orc") + + sql """set force_jni_scanner=true""" + order_qt_precision_evolution_jni_parquet precisionEvolutionQuery( + "timestamp_precision_evolution_parquet") + order_qt_precision_evolution_jni_orc precisionEvolutionQuery( + "timestamp_precision_evolution_orc") + order_qt_precision_evolution_jni_parquet_predicate precisionEvolutionPredicateQuery( + "timestamp_precision_evolution_parquet") + order_qt_precision_evolution_jni_orc_predicate precisionEvolutionPredicateQuery( + "timestamp_precision_evolution_orc") + // Each fixture has one historical p6 value at .000001 and a current p4 schema. + // Equality at the p4 value ensures Paimon does not prune the split from p6 stats. + order_qt_leading_zero_p4_jni_parquet """ + select id, cast(ts as string), microsecond(ts) + from doris_29023_timestamp_leading_zero_parquet + where ts = '2025-01-01 00:00:01.0000' + order by id + """ + order_qt_leading_zero_p4_jni_orc """ + select id, cast(ts as string), microsecond(ts) + from doris_29023_timestamp_leading_zero_orc + where ts = '2025-01-01 00:00:01.0000' + order by id + """ + // The p4 trigger column activates repair; projecting only the p6 timestamp-key map + // verifies that JNI can build the reader without widening the map key type. + order_qt_timestamp_map_key_jni_reader """ + select id, cmap + from doris_29023_timestamp_map_key_parquet + order by id + """ + order_qt_timestamp_map_key_jni_lookup """ + select cmap['2025-01-01 00:00:01.000001'] + from doris_29023_timestamp_map_key_parquet + order by id + """ + // Both nested fields evolved from p6 to p4. Only those fields are projected, and the + // negative-epoch fractional values exercise recursive JNI repair and its residual filter. + order_qt_timestamp_nested_jni_projection """ + select nested_array, nested_row + from doris_29023_timestamp_nested_parquet + order by nested_array[1] + """ + order_qt_timestamp_nested_jni_predicate """ + select nested_array[1], element_at(nested_row, 'nested_ts') + from doris_29023_timestamp_nested_parquet + where nested_array[1] = '1969-12-31 23:59:59.0000' + order by nested_array[1] + """ + + // Historical Parquet p6 values evolve across timestamp families to p0 in both + // directions. Control columns make the same family change at p6, preserving the + // SDK's JVM-zone conversion without precision loss. Compare against these controls + // so expected results do not depend on the community worker's default JVM timezone. + sql """set force_jni_scanner=true""" + for (def zone : ["UTC", "Asia/Shanghai"]) { + sql "set time_zone = '${zone}'" + def zoneTag = zone.replace('/', '_') + for (def forceJni : [true, false]) { + sql "set force_jni_scanner = ${forceJni}" + def modeTag = forceJni ? "" : "_automatic" + "order_qt_cross_family_${zoneTag}${modeTag}_projection" """ + select id, + ntz_to_ltz = date_trunc(ntz_to_ltz_control, 'second'), + ltz_to_ntz = date_trunc(ltz_to_ntz_control, 'second'), + microsecond(ntz_to_ltz), microsecond(ltz_to_ntz) + from timestamp_cross_family_evolution_parquet order by id + """ + for (def direction : ["ntz_to_ltz", "ltz_to_ntz"]) { + // Use SDK-converted controls for literals even when automatic routing is tested; + // unchanged cross-family native casts have their own session-zone semantics. + sql "set force_jni_scanner = true" + def literals = sql """select id, + cast(date_trunc(${direction}_control, 'second') as string) + from timestamp_cross_family_evolution_parquet order by id + """ + sql "set force_jni_scanner = ${forceJni}" + for (def row : literals) { + "order_qt_cross_family_${zoneTag}${modeTag}_${direction}_eq_${row[0]}" """ + select id from timestamp_cross_family_evolution_parquet + where ${direction} = '${row[1]}' order by id + """ + } + } + } + } + sql """set time_zone = 'Asia/Shanghai'""" sql """set force_jni_scanner=true""" test_scale() // test_ltz_ntz("test_timestamp_ntz_ltz_orc") @@ -194,8 +335,8 @@ suite("paimon_timestamp_types", "p0,external") { } } } finally { - sql """set force_jni_scanner=${originalForceJni}""" - sql """set enable_file_scanner_v2=${originalScannerV2}""" + sql """set enable_file_scanner_v2=${originalSettings.enable_file_scanner_v2}""" + sql """set force_jni_scanner=${originalSettings.force_jni_scanner}""" sql """set time_zone = '${originalTimeZone}'""" }