From fe45d971ea2e7476067e4f05b25b3082768ed7c4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 12 Oct 2026 13:12:01 +0800 Subject: [PATCH 01/13] [feat](datalake) Support nanosecond timestamps in Iceberg and Paimon --- .../data_type_timestamp_ns_serde.cpp | 181 ++++- .../data_type_timestamp_ns_serde.h | 9 + .../core/data_type_serde/orc_serde_utils.cpp | 13 + be/src/core/data_type_serde/orc_serde_utils.h | 3 + .../core/data_type_serde/parquet_timestamp.h | 31 + .../writer/iceberg/partition_transformers.cpp | 17 + .../writer/iceberg/partition_transformers.h | 7 + .../iceberg/viceberg_partition_writer.cpp | 37 +- .../writer/iceberg/viceberg_table_writer.cpp | 2 + be/src/format/orc/vorc_reader.cpp | 13 + .../format/parquet/parquet_column_convert.cpp | 2 + .../format/parquet/parquet_column_convert.h | 13 + be/src/format/parquet/schema_desc.cpp | 5 + .../table/iceberg/arrow_schema_util.cpp | 3 +- be/src/format/table/iceberg/types.cpp | 4 + be/src/format/table/iceberg/types.h | 12 +- .../format/transformer/vorc_transformer.cpp | 21 +- be/src/format_v2/orc/orc_reader.cpp | 19 +- .../format_v2/parquet/native_schema_desc.cpp | 5 + be/src/format_v2/parquet/parquet_reader.cpp | 10 +- be/src/format_v2/parquet/parquet_scan.cpp | 1 + be/src/format_v2/table/iceberg_reader.cpp | 2 +- .../data_type/data_type_timestamp_ns_test.cpp | 42 +- .../data_type_serde_parquet_test.cpp | 53 ++ .../iceberg/partition_transformers_test.cpp | 10 +- .../table/iceberg/arrow_schema_util_test.cpp | 11 + .../transformer/vorc_transformer_test.cpp | 44 ++ .../transformer/vparquet_writer_test.cpp | 51 ++ .../format_v2/parquet/parquet_reader_test.cpp | 47 ++ .../doris/paimon/PaimonArrowBatchAdapter.java | 4 + .../paimon/PaimonArrowBatchAdapterTest.java | 26 +- .../iceberg/IcebergPartitionUtils.java | 24 + .../connector/iceberg/IcebergSchemaUtils.java | 8 +- .../connector/iceberg/IcebergTypeMapping.java | 8 +- .../iceberg/IcebergTimestampNsTest.java | 79 +++ .../iceberg/IcebergTypeMappingReadTest.java | 4 +- .../paimon/PaimonScanPlanProvider.java | 25 +- .../connector/paimon/PaimonTypeMapping.java | 6 + ...nConnectorMetadataColumnEvolutionTest.java | 7 +- .../paimon/PaimonTimestampMappingTest.java | 5 +- .../paimon/PaimonTimestampNsTest.java | 59 ++ .../iceberg/test_iceberg_timestamp_ns.out | 407 +++++++++++ .../paimon/paimon_timestamp_types.out | 14 +- .../paimon/test_paimon_timestamp_ns.out | 659 ++++++++++++++++++ .../iceberg/test_iceberg_timestamp_ns.groovy | 92 +++ .../paimon/paimon_timestamp_types.groovy | 22 +- .../paimon/test_paimon_timestamp_ns.groovy | 90 +++ 47 files changed, 2147 insertions(+), 60 deletions(-) create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampNsTest.java create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index abe2db517f3ba7..96376a9e6911c0 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -28,6 +28,9 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_timestamp_ns.h" #include "util/mysql_row_buffer.h" @@ -37,6 +40,56 @@ namespace doris { namespace { +class TimestampNsParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimestampNsParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t count, size_t width) override { + const auto old_size = _data.size(); + for (size_t i = 0; i < count; ++i) { + int64_t nanos; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_nanos( + unaligned_load(values + i * width), &nanos); + } else if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(width, sizeof(int64_t)); + status = parquet_timestamp_nanos( + _context.time_unit, unaligned_load(values + i * width), &nanos); + } else { + return Status::NotSupported("TIMESTAMP_NS expects Parquet INT64 or INT96"); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + _data.emplace_back(nanos); + } + return Status::OK(); + } + +private: + ColumnTimeStampNs::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimestampNsBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMP_NS"); + } +}; + Status get_nanos_per_arrow_timestamp_unit(arrow::TimeUnit::type unit, int64_t* nanos_per_unit) { switch (unit) { case arrow::TimeUnit::SECOND: @@ -345,6 +398,110 @@ Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector( source_null_map); } +Status DataTypeTimeStampNsSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampNsParquetConsumer consumer(column, context); + RejectTimestampNsBinaryConsumer binary; + return source.decode_dictionary(consumer, binary); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + TimestampNsParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampNsParquetConsumer dictionary(*state.typed_dictionary, context, &state); + RejectTimestampNsBinaryConsumer binary; + auto status = source.decode_dictionary(dictionary, binary); + state.end_dictionary_conversion(null_map); + RETURN_IF_ERROR(status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT64 && view.value_kind != DecodedValueKind::INT96) { + return Status::NotSupported("TIMESTAMP_NS expects decoded INT64 or INT96 values"); + } + if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { + return Status::Corruption("Decoded TIMESTAMP_NS value buffer is null"); + } + auto unit = ParquetTimeUnit::MICROS; + if (view.time_unit == DecodedTimeUnit::MILLIS) { + unit = ParquetTimeUnit::MILLIS; + } else if (view.time_unit == DecodedTimeUnit::NANOS) { + unit = ParquetTimeUnit::NANOS; + } + auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.emplace_back(); + continue; + } + int64_t nanos; + auto status = + view.value_kind == DecodedValueKind::INT96 + ? parquet_int96_timestamp_nanos( + unaligned_load( + view.values + row * sizeof(ParquetInt96Timestamp)), + &nanos) + : parquet_timestamp_nanos( + unit, + unaligned_load(view.values + row * sizeof(int64_t)), + &nanos); + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } + data.emplace_back(nanos); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + const auto& batch = assert_cast(*view.batch); + auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); + const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows); + const auto timezone = view.timezone == nullptr ? cctz::utc_time_zone() : *view.timezone; + for (size_t row = 0; row < rows; ++row) { + const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows); + if (orc_serde_utils::orc_row_is_null(batch, source)) { + data.emplace_back(); + continue; + } + const auto nanos = batch.nanoseconds[source]; + if (nanos < 0 || nanos >= TimeStampNsValue::NANOS_PER_SECOND) { + data.resize(old_size); + return Status::DataQualityError("Invalid ORC timestamp nanoseconds: {}", nanos); + } + DateV2Value civil; + auto status = orc_serde_utils::orc_timestamp_to_datetime(batch.data[source], nanos / 1000, + timezone, false, &civil); + TimeStampNsValue value; + if (!status.ok() || !value.from_datetime(civil, nanos % 1000)) { + data.resize(old_size); + return Status::DataQualityError("ORC timestamp is outside the TIMESTAMP_NS range"); + } + data.push_back(value); + } + return Status::OK(); +} + Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, @@ -438,7 +595,29 @@ Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone orc::ColumnVectorBatch* orc_col_batch, int64_t start, int64_t end, Arena& arena, const FormatOptions& options) const { - return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc"); + const auto& data = assert_cast(column).get_data(); + auto& batch = assert_cast(*orc_col_batch); + for (int64_t row = start; row < end; ++row) { + if (batch.notNull[row] == 0) { + continue; + } + int64_t seconds; + auto civil = data[row].to_datetime(); + if (!civil.unix_timestamp(&seconds, timezone)) { + return Status::DataQualityError("Cannot convert TIMESTAMP_NS to ORC timestamp"); + } + const auto nanos = data[row].nanosecond(); + // Match the other timestamp writers: ORC-645 aliases these fractions across the epoch. + if (seconds == -1 && nanos >= 1000000) { + return Status::NotSupported( + "ORC cannot represent pre-epoch timestamp fractions in [-0.999, 0) " + "seconds without data loss; use Parquet for these values"); + } + batch.data[row] = seconds; + batch.nanoseconds[row] = nanos; + } + batch.numElements = end - start; + return Status::OK(); } void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column, diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h index 8351aa96961b58..508078d43f0f80 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h @@ -78,6 +78,15 @@ class DataTypeTimeStampNsSerDe final ColumnNullable& column_to, const ColumnString& column_from, CastParameters& cast_params, const NullMap::value_type* source_null_map = nullptr) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, const cctz::time_zone& ctz) const override; diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp b/be/src/core/data_type_serde/orc_serde_utils.cpp index e423307122c7f5..e4d78c2b7a2901 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.cpp +++ b/be/src/core/data_type_serde/orc_serde_utils.cpp @@ -23,6 +23,19 @@ namespace doris::orc_serde_utils { +bool has_nanosecond_timestamp(const ::orc::Type& type) { + if (type.getKind() == ::orc::TIMESTAMP && type.hasAttributeKey("iceberg.timestamp-unit") && + type.getAttributeValue("iceberg.timestamp-unit") == "NANOS") { + return true; + } + for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { + if (has_nanosecond_timestamp(*type.getSubtype(child))) { + return true; + } + } + return false; +} + size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { if (selected_rows == nullptr) { return rows; diff --git a/be/src/core/data_type_serde/orc_serde_utils.h b/be/src/core/data_type_serde/orc_serde_utils.h index f26b5ad2aeaaa1..00a2debe1bb4bb 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.h +++ b/be/src/core/data_type_serde/orc_serde_utils.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -29,6 +30,8 @@ namespace doris { namespace orc_serde_utils { +bool has_nanosecond_timestamp(const ::orc::Type& type); + size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows); size_t orc_source_row_at(size_t row, const std::vector* selected_rows); bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row); diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h index ba2aa686ad4272..67e0095da7a102 100644 --- a/be/src/core/data_type_serde/parquet_timestamp.h +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -46,6 +46,37 @@ inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { return Status::OK(); } +inline Status parquet_timestamp_nanos(ParquetTimeUnit unit, int64_t value, int64_t* result) { + int64_t factor = 1000; + if (unit == ParquetTimeUnit::MILLIS) { + factor = 1000000; + } else if (unit == ParquetTimeUnit::NANOS) { + factor = 1; + } + if (__builtin_mul_overflow(value, factor, result)) { + return Status::DataQualityError("Parquet timestamp is outside the TIMESTAMP_NS range"); + } + return Status::OK(); +} + +inline Status parquet_int96_timestamp_nanos(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + // Widen before multiplying: the first representable day's midnight is outside Int64. + const __int128 nanos = + (static_cast<__int128>(value.julian_day) - 2440588) * NANOS_IN_DAY + value.nanos_of_day; + if (nanos < std::numeric_limits::min() || + nanos > std::numeric_limits::max()) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the TIMESTAMP_NS range"); + } + *result = static_cast(nanos); + return Status::OK(); +} + inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { if (unit == ParquetTimeUnit::MILLIS) { // Validate in the source unit before scaling; signed overflow could otherwise turn an diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp index 12673f952f1325..d1f141792cce81 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.cpp +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.cpp @@ -132,6 +132,9 @@ std::unique_ptr PartitionColumnTransforms::create( case TYPE_DATETIMEV2: return std::make_unique>( source_type, parsed_width); + case TYPE_TIMESTAMP_NS: + return std::make_unique>( + source_type, parsed_width); case TYPE_DECIMALV2: { return std::make_unique>( source_type, parsed_width); @@ -174,6 +177,9 @@ std::unique_ptr PartitionColumnTransforms::create( case TYPE_DATETIMEV2: return std::make_unique>( source_type); + case TYPE_TIMESTAMP_NS: + return std::make_unique>( + source_type); default: { throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Unsupported type {} for partition column transform {}", @@ -191,6 +197,9 @@ std::unique_ptr PartitionColumnTransforms::create( case TYPE_DATETIMEV2: return std::make_unique>( source_type); + case TYPE_TIMESTAMP_NS: + return std::make_unique>( + source_type); default: { throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Unsupported type {} for partition column transform {}", @@ -208,6 +217,9 @@ std::unique_ptr PartitionColumnTransforms::create( case TYPE_DATETIMEV2: return std::make_unique>( source_type); + case TYPE_TIMESTAMP_NS: + return std::make_unique>( + source_type); default: { throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Unsupported type {} for partition column transform {}", @@ -222,6 +234,9 @@ std::unique_ptr PartitionColumnTransforms::create( case TYPE_DATETIMEV2: return std::make_unique>( source_type); + case TYPE_TIMESTAMP_NS: + return std::make_unique>( + source_type); default: { throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Unsupported type {} for partition column transform {}", @@ -307,6 +322,8 @@ std::string PartitionColumnTransform::get_partition_value(const DataTypePtr type return std::any_cast(value).to_string(cctz::utc_time_zone(), type->get_scale()); } + case TYPE_TIMESTAMP_NS: + return std::any_cast(value).to_string(); case TYPE_DATETIMEV2: { char buf[64]; char* pos = std::any_cast>(value).to_string( diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.h b/be/src/exec/sink/writer/iceberg/partition_transformers.h index 9354e70c76b69a..57097d085f0270 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.h @@ -21,6 +21,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_varbinary.h" #include "core/data_type/data_type_factory.hpp" +#include "core/value/timestamp_ns_value.h" #include "core/value/uuid_value.h" #include "exec/common/stringop_substring.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" @@ -76,6 +77,12 @@ class PartitionColumnTransformUtils { return value.utc_dt(); } + static DateV2Value timestamp_value(const TimeStampNsValue& value) { + // Iceberg bucket hashing uses epoch microseconds even for timestamp_ns; calendar + // transforms also ignore sub-microsecond digits. to_datetime floors negative fractions. + return value.to_datetime(); + } + static std::string human_year(int year_ordinal) { auto ymd = std::chrono::year_month_day {EPOCH} + std::chrono::years(year_ordinal); return std::to_string(static_cast(ymd.year())); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index a4bacb3b01d28d..104c1e91cb714c 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -30,6 +30,31 @@ #include "runtime/runtime_state.h" namespace doris { +namespace { +bool contains_nanosecond_timestamp(const iceberg::Type& type) { + switch (type.type_id()) { + case iceberg::TypeID::TIMESTAMP: + return static_cast(type).is_nanosecond(); + case iceberg::TypeID::STRUCT: + for (const auto& field : static_cast(type).fields()) { + if (contains_nanosecond_timestamp(*field.field_type())) { + return true; + } + } + return false; + case iceberg::TypeID::LIST: + return contains_nanosecond_timestamp( + *static_cast(type).element_field().field_type()); + case iceberg::TypeID::MAP: { + const auto& map = static_cast(type); + return contains_nanosecond_timestamp(*map.key_field().field_type()) || + contains_nanosecond_timestamp(*map.value_field().field_type()); + } + default: + return false; + } +} +} // namespace VIcebergPartitionWriter::VIcebergPartitionWriter( const TDataSink& t_sink, std::vector partition_values, @@ -84,10 +109,14 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil open_status = parquet_compression_type.error(); break; } - ParquetFileOptions parquet_options = {.compression_type = parquet_compression_type.value(), - .parquet_version = TParquetVersion::PARQUET_1_0, - .parquet_disable_dictionary = false, - .enable_int96_timestamps = false}; + // Parquet 1.0 coerces Arrow nanoseconds to microseconds, including nested fields. + ParquetFileOptions parquet_options = { + .compression_type = parquet_compression_type.value(), + .parquet_version = contains_nanosecond_timestamp(_schema.root_struct()) + ? TParquetVersion::PARQUET_2_LATEST + : TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; _file_format_transformer = std::make_unique( state, _file_writer.get(), _write_output_expr_ctxs, _write_column_names, false, parquet_options, _iceberg_schema_json, _schema, _nan_count_field_ids); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index c047f5fc88efe6..c041ac9e662046 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -963,6 +963,8 @@ std::any VIcebergTableWriter::_get_iceberg_partition_value( case TYPE_DATETIMEV2: { return binary_cast>(*(int64_t*)item); } + case TYPE_TIMESTAMP_NS: + return assert_cast(*col_ptr).get_data()[position]; case TYPE_TIMESTAMPTZ: { // Keep the UTC carrier typed through identity routing and FE commit serialization. return assert_cast(*col_ptr).get_data()[position]; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 958ed0f3ca96a2..c6544cb178a0dd 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -29,6 +29,7 @@ #include #include +#include "core/data_type_serde/orc_serde_utils.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vexpr.h" @@ -418,6 +419,12 @@ Status OrcReader::_create_file_reader() { try { _reader = orc::createReader( std::unique_ptr(_file_input_stream.release()), options); + if (orc_serde_utils::has_nanosecond_timestamp(_reader->getType())) { + // ORC and cctz disagree on pre-transition local offsets. UTC preserves the + // wall-clock value across the full nanosecond range without that conversion. + _ctz = "GMT"; + _time_zone = cctz::utc_time_zone(); + } } catch (std::exception& e) { // invoker maybe just skip Status.NotFound and continue // so we need distinguish between it and other kinds of errors @@ -1598,6 +1605,10 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); } case orc::TypeKind::TIMESTAMP: + if (orc_type->hasAttributeKey("iceberg.timestamp-unit") && + orc_type->getAttributeValue("iceberg.timestamp-unit") == "NANOS") { + return DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, true); + } return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, true, 0, 6); case orc::TypeKind::DECIMAL: @@ -1948,12 +1959,14 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, #undef DISPATCH case PrimitiveType::TYPE_INT: return _decode_int32_column(col_name, data_column, cvb, num_values); + case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_UUID: { OrcDecodedColumnView view; view.file_type = orc_column_type; view.selected_type = orc_column_type; view.batch = cvb; view.rows = num_values; + view.timezone = &_time_zone; return data_type->get_serde()->read_column_from_orc(*data_column, view); } case PrimitiveType::TYPE_DECIMAL32: diff --git a/be/src/format/parquet/parquet_column_convert.cpp b/be/src/format/parquet/parquet_column_convert.cpp index c0fbadbc6d56da..65f69bc9aca8c0 100644 --- a/be/src/format/parquet/parquet_column_convert.cpp +++ b/be/src/format/parquet/parquet_column_convert.cpp @@ -308,6 +308,8 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv convert_params.get(), physical_converter); } else if (src_logical_primitive == TYPE_DATEV2) { physical_converter = std::make_unique(); + } else if (src_logical_primitive == TYPE_TIMESTAMP_NS) { + physical_converter = std::make_unique(); } else if (src_logical_primitive == TYPE_DATETIMEV2) { if (src_physical_type == tparquet::Type::INT96) { // int96 only stores nanoseconds in standard parquet file diff --git a/be/src/format/parquet/parquet_column_convert.h b/be/src/format/parquet/parquet_column_convert.h index 083711a5ce34a3..5a1c8b7260e1b1 100644 --- a/be/src/format/parquet/parquet_column_convert.h +++ b/be/src/format/parquet/parquet_column_convert.h @@ -836,6 +836,19 @@ class Int32ToDate : public PhysicalToLogicalConverter { } }; +struct Int64ToTimestampNs : public PhysicalToLogicalConverter { + Status physical_convert(ColumnPtr& source, ColumnPtr& target) override { + const auto& input = assert_cast(*remove_nullable(source)).get_data(); + auto& output = + assert_cast(*get_mutable_inner_column(target)).get_data(); + // Only the unadjusted TIMESTAMP(NANOS) annotation selects this converter. + for (int64_t nanos : input) { + output.emplace_back(nanos); + } + return Status::OK(); + } +}; + struct Int64ToTimestamp : public PhysicalToLogicalConverter { Status physical_convert(ColumnPtr& src_physical_col, ColumnPtr& src_logical_column) override { ColumnPtr src_col = remove_nullable(src_physical_col); diff --git a/be/src/format/parquet/schema_desc.cpp b/be/src/format/parquet/schema_desc.cpp index c2c0ed22ed33df..1f22091df9c8f8 100644 --- a/be/src/format/parquet/schema_desc.cpp +++ b/be/src/format/parquet/schema_desc.cpp @@ -302,6 +302,11 @@ std::pair FieldDescriptor::convert_to_doris_type( } else if (logicalType.__isset.TIME) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); } else if (logicalType.__isset.TIMESTAMP) { + if (logicalType.TIMESTAMP.unit.__isset.NANOS && !logicalType.TIMESTAMP.isAdjustedToUTC) { + // Materialize nanoseconds before any table cast; DATETIMEV2 would discard the remainder. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, nullable); + return ans; + } if (_enable_mapping_timestamp_tz) { if (logicalType.TIMESTAMP.isAdjustedToUTC) { // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index 42061eedea692d..557a813d1ea18f 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -87,7 +87,8 @@ Status IcebergArrowSchemaConvertor::convert_to_arrow_field( case iceberg::TypeID::TIMESTAMP: { iceberg::TimestampType* t_type = static_cast(field.field_type()); std::string real_tz = t_type->should_adjust_to_utc() ? _timezone : ""; - arrow_type = std::make_shared(arrow::TimeUnit::MICRO, real_tz); + arrow_type = std::make_shared( + t_type->is_nanosecond() ? arrow::TimeUnit::NANO : arrow::TimeUnit::MICRO, real_tz); break; } diff --git a/be/src/format/table/iceberg/types.cpp b/be/src/format/table/iceberg/types.cpp index 252f9035518c0b..cfdfb7479f4050 100644 --- a/be/src/format/table/iceberg/types.cpp +++ b/be/src/format/table/iceberg/types.cpp @@ -164,6 +164,10 @@ std::unique_ptr Types::from_primitive_string(const std::string& t return std::make_unique(true); } else if (lower_type_string == "timestamp") { return std::make_unique(false); + } else if (lower_type_string == "timestamp_ns") { + return std::make_unique(false, true); + } else if (lower_type_string == "timestamptz_ns") { + return std::make_unique(true, true); } else if (lower_type_string == "string") { return std::make_unique(); } else if (lower_type_string == "uuid") { diff --git a/be/src/format/table/iceberg/types.h b/be/src/format/table/iceberg/types.h index 1594003fccda68..a4fb844897deee 100644 --- a/be/src/format/table/iceberg/types.h +++ b/be/src/format/table/iceberg/types.h @@ -317,22 +317,24 @@ class TimestampType : public PrimitiveType { public: ~TimestampType() override = default; - TimestampType(bool adjust_to_utc) : _adjust_to_utc(adjust_to_utc) {} + TimestampType(bool adjust_to_utc, bool nanoseconds = false) + : _adjust_to_utc(adjust_to_utc), _nanoseconds(nanoseconds) {} TypeID type_id() const override { return TypeID::TIMESTAMP; } bool should_adjust_to_utc() const { return _adjust_to_utc; } + bool is_nanosecond() const { return _nanoseconds; } std::string to_string() const override { - if (should_adjust_to_utc()) { - return "timestamptz"; - } else { - return "timestamp"; + if (_nanoseconds) { + return should_adjust_to_utc() ? "timestamptz_ns" : "timestamp_ns"; } + return should_adjust_to_utc() ? "timestamptz" : "timestamp"; } private: bool _adjust_to_utc; + bool _nanoseconds; }; class TimeType : public PrimitiveType { diff --git a/be/src/format/transformer/vorc_transformer.cpp b/be/src/format/transformer/vorc_transformer.cpp index dfb771979e8e21..be2f619d4c0910 100644 --- a/be/src/format/transformer/vorc_transformer.cpp +++ b/be/src/format/transformer/vorc_transformer.cpp @@ -47,6 +47,7 @@ #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/pod_array.h" #include "core/string_ref.h" #include "core/types.h" @@ -243,6 +244,11 @@ Status VOrcTransformer::open() { } } + if (orc_serde_utils::has_nanosecond_timestamp(*_schema)) { + // ORC and cctz use different ancient local offsets. A UTC wall-clock carrier avoids + // shifting nanosecond boundary dates when another session reads the file. + _write_options->setTimezoneName("GMT"); + } _output_stream = std::make_unique(_file_writer); try { _write_options->setMemoryPool(ExecEnv::GetInstance()->orc_memory_pool()); @@ -349,6 +355,11 @@ std::unique_ptr VOrcTransformer::_build_orc_type( type = orc::createPrimitiveType(orc::DATE); break; } + case TYPE_TIMESTAMP_NS: { + type = orc::createPrimitiveType(orc::TIMESTAMP); + type->setAttribute("iceberg.timestamp-unit", "NANOS"); + break; + } case TYPE_DATETIMEV2: { type = orc::createPrimitiveType(orc::TIMESTAMP); break; @@ -637,6 +648,10 @@ bool VOrcTransformer::_collect_column_bounds(const orc::ColumnStatistics* col_st } } else if (const auto* ts_stats = dynamic_cast(col_stats)) { + if (remove_nullable(data_type)->get_primitive_type() == TYPE_TIMESTAMP_NS) { + // Omit bounds rather than publish microseconds as Iceberg nanoseconds. + return false; + } if (ts_stats->hasMinimum() && ts_stats->hasMaximum()) { // ORC stores milliseconds plus a sub-millisecond nanos tail. Dropping that tail // makes Iceberg's upper bound smaller than real rows and causes false-negative pruning. @@ -741,9 +756,9 @@ Status VOrcTransformer::write(const Block& block) { normalized_columns.push_back(std::move(raw_column)); const auto& write_column = normalized_columns.back(); RETURN_IF_ERROR(_resize_row_batch(col.type, *write_column, root->fields[i])); - RETURN_IF_ERROR(_serdes[i]->write_column_to_orc(_state->timezone(), *write_column, - nullptr, root->fields[i], 0, sz, arena, - options)); + RETURN_IF_ERROR(_serdes[i]->write_column_to_orc(_write_options->getTimezoneName(), + *write_column, nullptr, root->fields[i], + 0, sz, arena, options)); } root->numElements = sz; _writer->add(*row_batch); diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 086e516e059ced..ad64e8dab7a9e7 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -62,6 +62,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_uuid.h" #include "core/data_type/data_type_varbinary.h" @@ -659,6 +660,11 @@ bool build_zone_map_from_orc_statistics(const ::orc::Type& type, case ::orc::TypeKind::DATE: return set_date_zone_map(statistics, zone_map); case ::orc::TypeKind::TIMESTAMP: { + if (type.hasAttributeKey("iceberg.timestamp-unit") && + type.getAttributeValue("iceberg.timestamp-unit") == "NANOS") { + // The microsecond statistics carrier cannot safely prune nanosecond values. + return false; + } // ORC stores timestamp statistics as wall-clock values in UTC coordinates. Restore them // with UTC so the session timezone does not shift the civil time. static const auto utc_time_zone = cctz::utc_time_zone(); @@ -947,6 +953,12 @@ Status OrcReader::init(RuntimeState* state) { try { _state->reader = ::orc::createReader(std::move(input_stream), options); _state->root_type = &_state->reader->getType(); + if (orc_serde_utils::has_nanosecond_timestamp(*_state->root_type)) { + // Use the same UTC wall-clock carrier as the writer; historical local offsets + // differ between ORC and cctz and can push boundary values out of range. + _state->timezone = "GMT"; + _state->timezone_obj = cctz::utc_time_zone(); + } } catch (const std::exception& e) { if (is_orc_stop(_io_ctx.get(), e)) { return Status::EndOfFile("stop"); @@ -1062,7 +1074,12 @@ DataTypePtr OrcReader::_convert_to_doris_type(const ::orc::Type& type) const { data_type = std::make_shared(); break; case ::orc::TypeKind::TIMESTAMP: - data_type = std::make_shared(6); + if (type.hasAttributeKey("iceberg.timestamp-unit") && + type.getAttributeValue("iceberg.timestamp-unit") == "NANOS") { + data_type = std::make_shared(); + } else { + data_type = std::make_shared(6); + } break; case ::orc::TypeKind::TIMESTAMP_INSTANT: if (_enable_mapping_timestamp_tz) { diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index b34b56c4c62d39..19180c5c86c095 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -882,6 +882,11 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( // sub-second values are silently truncated by the target SerDe. ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); } else if (logicalType.__isset.TIMESTAMP) { + if (logicalType.TIMESTAMP.unit.__isset.NANOS && !logicalType.TIMESTAMP.isAdjustedToUTC) { + // Materialize nanoseconds before any table cast; DATETIMEV2 would discard the remainder. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, nullable); + return ans; + } if (_enable_mapping_timestamp_tz) { if (logicalType.TIMESTAMP.isAdjustedToUTC) { // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index b087c3a82f975b..a94b7dae4cad7e 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -366,8 +366,14 @@ DataTypePtr apply_projection_timestamp_semantics(ParquetColumnSchema* column_sch const bool physical_timestamp = descriptor.physical_type == tparquet::Type::INT96 || descriptor.is_timestamp; if (physical_timestamp && projection.timestamp_is_adjusted_to_utc.has_value()) { - const auto target = - *projection.timestamp_is_adjusted_to_utc ? TYPE_TIMESTAMPTZ : TYPE_DATETIMEV2; + // A table's wall-clock annotation must not narrow a nanosecond physical column. + auto target = TYPE_DATETIMEV2; + if (*projection.timestamp_is_adjusted_to_utc) { + target = TYPE_TIMESTAMPTZ; + } else if (column_schema->type != nullptr && + column_schema->type->get_primitive_type() == TYPE_TIMESTAMP_NS) { + target = TYPE_TIMESTAMP_NS; + } column_schema->type = DataTypeFactory::instance().create_data_type( target, column_schema->type != nullptr && column_schema->type->is_nullable(), 0, timestamp_tz_scale(descriptor)); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index aa4368197a7f4d..ac580eea37b8d0 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -1579,6 +1579,7 @@ bool get_typed_dictionary_raw_values(PrimitiveType primitive_type, const IColumn GET_TYPED_DICTIONARY_VALUES(TYPE_DATEV2); GET_TYPED_DICTIONARY_VALUES(TYPE_DATETIMEV2); GET_TYPED_DICTIONARY_VALUES(TYPE_TIMESTAMPTZ); + GET_TYPED_DICTIONARY_VALUES(TYPE_TIMESTAMP_NS); GET_TYPED_DICTIONARY_VALUES(TYPE_DECIMAL32); GET_TYPED_DICTIONARY_VALUES(TYPE_DECIMAL64); GET_TYPED_DICTIONARY_VALUES(TYPE_DECIMALV2); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 1ac44f417663a2..6af63be1374e0a 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -305,7 +305,7 @@ static std::string iceberg_json_scalar_text(const rapidjson::Value& value) { static void normalize_iceberg_json_timestamp(PrimitiveType primitive_type, std::string* value) { if (primitive_type != TYPE_DATETIME && primitive_type != TYPE_DATETIMEV2 && - primitive_type != TYPE_TIMESTAMPTZ) { + primitive_type != TYPE_TIMESTAMPTZ && primitive_type != TYPE_TIMESTAMP_NS) { return; } if (const size_t separator = value->find('T'); separator != std::string::npos) { diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp index c0e30f314ae7aa..6496cb19af93d6 100644 --- a/be/test/core/data_type/data_type_timestamp_ns_test.cpp +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -464,16 +466,18 @@ TEST(DataTypeTimeStampNsTest, SerDeRoundTripsTextProtobufAndBinary) { EXPECT_EQ(binary_data, source_data); } -TEST(DataTypeTimeStampNsTest, SerDeExplicitlyRejectsUnsupportedOrcIo) { +TEST(DataTypeTimeStampNsTest, SerDeWritesEmptyOrcBatch) { const DataTypeTimeStampNs type; const auto serde = type.get_serde(); auto column = type.create_column(); Arena arena; DataTypeSerDe::FormatOptions options; + orc::TimestampVectorBatch batch(0, *orc::getDefaultPool()); const auto orc_write_status = - serde->write_column_to_orc("UTC", *column, nullptr, nullptr, 0, 0, arena, options); - EXPECT_TRUE(orc_write_status.is()) << orc_write_status; + serde->write_column_to_orc("UTC", *column, nullptr, &batch, 0, 0, arena, options); + EXPECT_TRUE(orc_write_status.ok()) << orc_write_status; + EXPECT_EQ(0, batch.numElements); } TEST(DataTypeTimeStampNsTest, DataTypeLiteralField) { @@ -682,4 +686,36 @@ TEST(DataTypeTimeStampNsTest, SerDeBatchRejectsMultipleFractionalSeparators) { .ok()); } +TEST(DataTypeTimeStampNsTest, OrcRoundTripPreservesNanoseconds) { + DataTypeTimeStampNs type; + auto serde = type.get_serde(); + auto input = type.create_column(); + auto& values = assert_cast(*input).get_data(); + for (int64_t nanos : {-1000000001LL, 0LL, 1LL, 123456789LL}) { + values.emplace_back(nanos); + } + orc::TimestampVectorBatch batch(4, *orc::getDefaultPool()); + for (size_t i = 0; i < 4; ++i) { + batch.notNull[i] = 1; + } + Arena arena; + DataTypeSerDe::FormatOptions options; + ASSERT_TRUE( + serde->write_column_to_orc("UTC", *input, nullptr, &batch, 0, 4, arena, options).ok()); + auto orc_type = orc::createPrimitiveType(orc::TIMESTAMP); + auto output = type.create_column(); + auto timezone = cctz::utc_time_zone(); + OrcDecodedColumnView view; + view.file_type = orc_type.get(); + view.batch = &batch; + view.rows = 4; + view.timezone = &timezone; + ASSERT_TRUE(serde->read_column_from_orc(*output, view).ok()); + const auto& result = assert_cast(*output).get_data(); + ASSERT_EQ(result.size(), values.size()); + for (size_t i = 0; i < result.size(); ++i) { + EXPECT_EQ(result[i].epoch_nanos(), values[i].epoch_nanos()); + } +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index c6ea8c24ebd090..3d49af3bd5c9cc 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -33,6 +33,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_uuid.h" #include "core/data_type/data_type_varbinary.h" @@ -1150,4 +1151,56 @@ TEST(DataTypeSerDeParquetTest, UuidPlainAndDictionaryPreserveBigEndianBits) { } } // namespace +TEST(DataTypeSerDeParquetTest, TimestampNsPreservesSignedNanosecondsAndDictionary) { + DataTypeTimeStampNs type; + auto serde = type.get_serde(); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT64; + context.time_unit = ParquetTimeUnit::NANOS; + const std::vector values = {std::numeric_limits::min(), -1, 0, 1, 123456789, + std::numeric_limits::max()}; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary(values, {0, 1, 2, 3, 4, 5}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values(values); + } + auto column = type.create_column(); + ParquetMaterializationState state; + ASSERT_TRUE(serde->read_column_from_parquet(*column, source, context, values.size(), state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), values.size()); + for (size_t i = 0; i < values.size(); ++i) { + EXPECT_EQ(data[i].epoch_nanos(), values[i]); + } + } +} + +TEST(DataTypeSerDeParquetTest, TimestampNsReadsInt96AndRejectsOverflow) { + DataTypeTimeStampNs type; + auto serde = type.get_serde(); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT96; + TestParquetDecodeSource source; + source.set_fixed_values( + {{.nanos_of_day = NANOS_PER_DAY - 1, .julian_day = JULIAN_UNIX_EPOCH - 1}, + {.nanos_of_day = 1, .julian_day = JULIAN_UNIX_EPOCH}}); + auto column = type.create_column(); + ParquetMaterializationState state; + ASSERT_TRUE(serde->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].epoch_nanos(), -1); + EXPECT_EQ(data[1].epoch_nanos(), 1); + + TestParquetDecodeSource overflow; + overflow.set_fixed_values({std::numeric_limits::max()}); + context.physical_type = ParquetPhysicalType::INT64; + context.time_unit = ParquetTimeUnit::MICROS; + EXPECT_FALSE(serde->read_column_from_parquet(*column, overflow, context, 1, state).ok()); + EXPECT_EQ(column->size(), 2); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp index 2b85caa178eddb..d73b7dd7344e43 100644 --- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp +++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp @@ -22,6 +22,7 @@ #include #include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_uuid.h" #include "core/data_type/data_type_varbinary.h" @@ -146,17 +147,24 @@ TEST_F(PartitionTransformersTest, binary_truncate_counts_bytes_and_preserves_typ TEST_F(PartitionTransformersTest, timestamp_transforms_use_utc_calendar_and_microseconds) { for (const auto& type : std::vector {std::make_shared(6), - std::make_shared(6)}) { + std::make_shared(6), + std::make_shared()}) { auto column = type->create_column(); // The first two UTC values represent the repeated New York 01:30 with different offsets. const std::vector> fields = {{2021, 11, 7, 5, 30, 0, 123456}, {2021, 11, 7, 6, 30, 0, 123456}, {1969, 12, 31, 23, 59, 59, 999999}}; const std::vector micros = {1636263000123456, 1636266600123456, -1}; + size_t index = 0; for (const auto& f : fields) { DateV2Value dt; ASSERT_TRUE(dt.check_range_and_set_time(f[0], f[1], f[2], f[3], f[4], f[5], f[6])); auto packed = dt.to_date_int_val(); + if (type->get_primitive_type() == TYPE_TIMESTAMP_NS) { + // All sub-microsecond fractions must bucket using the floor microsecond. + packed = static_cast(micros[index] * 1000 + 999); + } + ++index; column->insert_data(reinterpret_cast(&packed), sizeof(packed)); } column->insert_default(); diff --git a/be/test/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/format/table/iceberg/arrow_schema_util_test.cpp index ce1d3e84123946..cdb3f4fc07cb66 100644 --- a/be/test/format/table/iceberg/arrow_schema_util_test.cpp +++ b/be/test/format/table/iceberg/arrow_schema_util_test.cpp @@ -54,6 +54,17 @@ class ArrowSchemaUtilTest : public testing::Test { virtual ~ArrowSchemaUtilTest() = default; }; +TEST(ArrowSchemaUtilTest, TimestampNsSchemaPreservesNanosecondUnit) { + const std::string json = R"({"type":"struct","fields":[ + {"id":1,"name":"ts","required":false,"type":"timestamp_ns"}]})"; + auto schema = SchemaParser::from_json(json); + IcebergArrowSchemaConvertor converter(*schema, "Asia/Shanghai", json); + std::shared_ptr arrow_schema; + ASSERT_TRUE(converter.get_arrow_schema(&arrow_schema).ok()); + EXPECT_TRUE(arrow_schema->field(0)->type()->Equals(arrow::timestamp(arrow::TimeUnit::NANO))); + EXPECT_EQ(schema->columns()[0].field_type()->to_string(), "timestamp_ns"); +} + TEST(ArrowSchemaUtilTest, IcebergParquetWriterPreservesUuidLogicalAnnotations) { auto* env = ExecEnv::GetInstance(); auto* previous_pool = env->_arrow_memory_pool; diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp index a0785522d822d9..0b933ba16ec2c5 100644 --- a/be/test/format/transformer/vorc_transformer_test.cpp +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -36,6 +36,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_uuid.h" #include "core/data_type/data_type_varbinary.h" @@ -90,6 +91,49 @@ TEST_F(VOrcTransformerTest, RejectsLossyPreEpochFractions) { } } +TEST_F(VOrcTransformerTest, NanosecondBoundsUseUtcWallClockCarrier) { + auto type = std::make_shared(); + auto exprs = MockSlotRef::create_mock_contexts(DataTypes {type}); + const std::string json = + R"({"type":"struct","fields":[{"id":1,"name":"ts","required":true,"type":"timestamp_ns"}]})"; + auto schema = iceberg::SchemaParser::from_json(json); + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("Asia/Shanghai"); + VOrcTransformer writer(&state, file_writer.get(), exprs, "", {"ts"}, false, + TFileCompressType::PLAIN, schema.get(), _fs); + ASSERT_TRUE(writer.open().ok()); + const std::vector nanos = { + std::numeric_limits::min(), -1000000001, 0, 1, 123456789, + std::numeric_limits::max()}; + auto column = type->create_column(); + for (auto value : nanos) { + column->insert(Field::create_field(TimeStampNsValue(value))); + } + ASSERT_TRUE(writer.write(Block({{std::move(column), type, "ts"}})).ok()); + ASSERT_TRUE(writer.close().ok()); + TIcebergColumnStats stats; + ASSERT_TRUE(writer.collect_file_statistics_after_close(&stats).ok()); + EXPECT_FALSE(stats.lower_bounds.contains(1)); + EXPECT_FALSE(stats.upper_bounds.contains(1)); + auto reader = orc::createReader(orc::readLocalFile(_file_path), orc::ReaderOptions()); + EXPECT_TRUE(orc_serde_utils::has_nanosecond_timestamp(reader->getType())); + orc::RowReaderOptions options; + options.setTimezoneName("GMT"); + auto rows = reader->createRowReader(options); + auto batch = rows->createRowBatch(nanos.size()); + ASSERT_TRUE(rows->next(*batch)); + const auto& values = assert_cast( + *assert_cast(*batch).fields[0]); + ASSERT_EQ(nanos.size(), batch->numElements); + for (size_t row = 0; row < nanos.size(); ++row) { + const __int128 actual = + static_cast<__int128>(values.data[row]) * 1000000000 + values.nanoseconds[row]; + EXPECT_EQ(static_cast<__int128>(nanos[row]), actual); + } +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity): assertions cover the timezone/type matrix. TEST_F(VOrcTransformerTest, TimestampBoundsRetainMicroseconds) { TimezoneUtils::load_timezones_to_cache(); diff --git a/be/test/format/transformer/vparquet_writer_test.cpp b/be/test/format/transformer/vparquet_writer_test.cpp index e7b446630f3683..d296f2c586d8b4 100644 --- a/be/test/format/transformer/vparquet_writer_test.cpp +++ b/be/test/format/transformer/vparquet_writer_test.cpp @@ -37,6 +37,8 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/vhive_partition_writer.h" #include "format/table/iceberg/schema_parser.h" #include "format/transformer/viceberg_parquet_writer.h" @@ -354,4 +356,53 @@ TEST_F(VParquetWriterTest, HiveInt96HonorsCatalogTimezoneContract) { } } +TEST_F(VParquetWriterTest, IcebergNanosecondsRoundTripWithoutCoercion) { + const auto type = make_nullable(std::make_shared()); + auto exprs = MockSlotRef::create_mock_contexts(DataTypes {type}); + const std::string json = + R"({"type":"struct","fields":[{"id":1,"name":"ts","required":false,"type":"timestamp_ns"}]})"; + auto schema = iceberg::SchemaParser::from_json(json); + const std::map hadoop_conf; + TDataSink sink; + RuntimeState state; + state.set_timezone("Asia/Shanghai"); + const auto name = "iceberg_ns_" + UniqueId::gen_uid().to_string(); + _file_path = "./" + name + "-0.parquet"; + VIcebergPartitionWriter writer(sink, {}, exprs, *schema, &json, {"ts"}, + {.write_path = ".", + .original_write_path = ".", + .target_path = ".", + .file_type = TFileType::FILE_LOCAL, + .broker_addresses = {}}, + name, 0, TFileFormatType::FORMAT_PARQUET, + TFileCompressType::PLAIN, hadoop_conf); + ASSERT_TRUE(writer.open(&state, nullptr, nullptr).ok()); + const std::vector nanos = {std::numeric_limits::min(), -1, 0, 1, 123456789, + std::numeric_limits::max()}; + auto column = type->create_column(); + for (auto value : nanos) { + column->insert(Field::create_field(TimeStampNsValue(value))); + } + column->insert_default(); + Block block({{std::move(column), type, "ts"}}); + auto status = writer.write(block); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(writer.close(Status::OK()).ok()); + auto input = arrow::io::ReadableFile::Open(_file_path); + ASSERT_TRUE(input.ok()) << input.status(); + auto reader = ::parquet::arrow::OpenFile(*input, arrow::default_memory_pool()); + ASSERT_TRUE(reader.ok()) << reader.status(); + auto table = (*reader)->ReadTable(); + ASSERT_TRUE(table.ok()) << table.status(); + const auto& values = assert_cast(*(*table)->column(0)->chunk(0)); + const auto& timestamp_type = assert_cast(*values.type()); + EXPECT_EQ(arrow::TimeUnit::NANO, timestamp_type.unit()); + EXPECT_TRUE(timestamp_type.timezone().empty()); + ASSERT_EQ(nanos.size() + 1, values.length()); + for (size_t row = 0; row < nanos.size(); ++row) { + EXPECT_EQ(nanos[row], values.Value(row)); + } + EXPECT_TRUE(values.IsNull(nanos.size())); +} + } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index ce8bd4f7b1f48b..37370b64218874 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -3001,6 +3001,53 @@ TEST_F(NewParquetReaderTest, Int96TimezoneUsesCatalogPropertyInsteadOfSessionTim EXPECT_EQ(read_first_value(std::string {}, true), "2024-12-31 16:00:00.000000"); } +TEST_F(NewParquetReaderTest, NanosecondsSurviveWallClockProjectionAndDictionary) { + const std::vector nanos = { + std::numeric_limits::min(), -1, 0, 1, 123456788, 123456789, + std::numeric_limits::max()}; + auto arrow_type = arrow::timestamp(arrow::TimeUnit::NANO); + auto array = build_timestamp_array(arrow_type, nanos); + auto table = arrow::Table::Make(arrow::schema({arrow::field("ts", arrow_type, true)}), {array}); + for (bool dictionary : {false, true}) { + auto file = arrow::io::FileOutputStream::Open(_file_path); + ASSERT_TRUE(file.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + if (dictionary) { + properties.enable_dictionary(); + } else { + properties.disable_dictionary(); + } + ASSERT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *file, + nanos.size(), properties.build()) + .ok()); + ASSERT_TRUE((*file)->Close().ok()); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("Asia/Shanghai"); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(1, schema.size()); + ASSERT_EQ(TYPE_TIMESTAMP_NS, schema[0].type->get_primitive_type()); + auto projection = field_projection(0); + projection.timestamp_is_adjusted_to_utc = false; + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + ASSERT_TRUE(reader->open(request).ok()); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(nanos.size(), rows); + const auto& values = nullable_nested_column(block, 0); + for (size_t row = 0; row < rows; ++row) { + EXPECT_EQ(nanos[row], values.get_element(row).epoch_nanos()); + } + } +} + TEST_F(NewParquetReaderTest, Int96UsesPerColumnTimestampSemantics) { write_int96_timestamp_parquet_file(_file_path); diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java index 918f47890737cf..1a4b0afb08bb09 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java @@ -204,6 +204,10 @@ private static FieldType timestampField( TimeUnit unit = precision > 3 ? TimeUnit.MICROSECOND : precision > 0 ? TimeUnit.MILLISECOND : TimeUnit.SECOND; + if (timeZone == null && precision > 6) { + // The transport must retain the sub-microsecond digits of TIMESTAMP_NS inputs. + unit = TimeUnit.NANOSECOND; + } return new FieldType(nullable, new ArrowType.Timestamp(unit, timeZone), null); } } diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java index 73c8685b20e12c..daaa2bb45ce1e4 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java @@ -174,9 +174,7 @@ public void testTargetSchemaDistinguishesNtzAndLtz() throws Exception { Assertions.assertNull(ntz.getTimezone()); Assertions.assertEquals(TimeUnit.MICROSECOND, ltz.getUnit()); Assertions.assertEquals("Asia/Shanghai", ltz.getTimezone()); - // Doris timestamps have microsecond precision, so a wider Paimon target still - // uses a microsecond transport instead of pretending the input contains nanos. - Assertions.assertEquals(TimeUnit.MICROSECOND, ntz9.getUnit()); + Assertions.assertEquals(TimeUnit.NANOSECOND, ntz9.getUnit()); Assertions.assertNull(ntz9.getTimezone()); } } @@ -199,13 +197,33 @@ public void testTimestampAdaptationDoesNotRepeatTimezoneConversion() throws Exce root.allocateNew(); ((TimeStampVector) root.getVector("ntz")).setSafe(0, ntzMicros); ((TimeStampVector) root.getVector("ltz")).setSafe(0, ltzInstantMicros); - ((TimeStampVector) root.getVector("ntz9")).setSafe(0, ntzMicros); + ((TimeStampVector) root.getVector("ntz9")).setSafe(0, ntzMicros * 1_000L + 789); root.setRowCount(1); InternalRow row = adapter.rows(root).row(0); Assertions.assertEquals(ntzMicros, row.getTimestamp(0, 6).toMicros()); Assertions.assertEquals(ltzInstantMicros, row.getTimestamp(1, 6).toMicros()); Assertions.assertEquals(ntzMicros, row.getTimestamp(2, 9).toMicros()); + Assertions.assertEquals(456789, row.getTimestamp(2, 9).getNanoOfMillisecond()); + } + } + } + + @Test + public void testNanosecondTransportPreservesSignedBounds() throws Exception { + RowType inputType = DataTypes.ROW(DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(9))); + try (RootAllocator allocator = new RootAllocator()) { + PaimonArrowBatchAdapter adapter = new PaimonArrowBatchAdapter(inputType, ZoneId.of("UTC"), allocator); + try (ArrowStreamReader reader = schemaReader(adapter, allocator)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + root.allocateNew(); + for (long nanos : new long[] {Long.MIN_VALUE, -1, 0, 1, Long.MAX_VALUE}) { + ((TimeStampVector) root.getVector("ts")).setSafe(0, nanos); + root.setRowCount(1); + org.apache.paimon.data.Timestamp value = adapter.rows(root).row(0).getTimestamp(0, 9); + Assertions.assertEquals(Math.floorDiv(nanos, 1_000_000L), value.getMillisecond()); + Assertions.assertEquals(Math.floorMod(nanos, 1_000_000L), value.getNanoOfMillisecond()); + } } } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index 127d9adebb969a..edce2f0463e3c0 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -399,6 +399,18 @@ static String serializePartitionValue(Type type, Object value, ZoneId zone) { // Iceberg time is stored as microseconds since midnight. long micros = (Long) value; return LocalTime.ofNanoOfDay(micros * 1000).format(DateTimeFormatter.ISO_LOCAL_TIME); + case TIMESTAMP_NANO: + if (value == null) { + return null; + } + long nanos = (Long) value; + LocalDateTime timestampNs = LocalDateTime.ofEpochSecond( + Math.floorDiv(nanos, 1_000_000_000L), + (int) Math.floorMod(nanos, 1_000_000_000L), ZoneOffset.UTC); + if (((org.apache.iceberg.types.Types.TimestampNanoType) type).shouldAdjustToUTC()) { + return timestampNs.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } + return timestampNs.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); case TIMESTAMP: if (value == null) { return null; @@ -479,6 +491,13 @@ static Object parsePartitionValueFromString(String valueStr, Type icebergType, Z case DATE: // Iceberg date is days since epoch (1970-01-01). return (int) LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay(); + case TIMESTAMP_NANO: + LocalDateTime timestampNs = LocalDateTime.parse(valueStr.replace('T', ' '), + TIMESTAMP_PARTITION_FORMAT); + // Widen before combining: the minimum nanosecond timestamp's whole second overflows long. + return BigDecimal.valueOf(timestampNs.toEpochSecond(ZoneOffset.UTC)) + .multiply(BigDecimal.valueOf(1_000_000_000L)) + .add(BigDecimal.valueOf(timestampNs.getNano())).longValueExact(); case TIMESTAMP: return parseTimestampToMicros(valueStr, (TimestampType) icebergType, zone); case DECIMAL: @@ -801,6 +820,11 @@ static boolean isValidRelatedTable(Table table) { return false; } PartitionField partitionField = fields.get(0); + if (spec.schema().findType(partitionField.sourceId()).typeId() == Type.TypeID.TIMESTAMP_NANO) { + // Calendar boundaries and the NULL sentinel can exceed TIMESTAMP_NS's domain. + // Leave pruning to Iceberg instead of building invalid Doris RANGE boundaries. + return false; + } String transformName = partitionField.transform().toString(); if (!YEAR.equals(transformName) && !MONTH.equals(transformName) && !DAY.equals(transformName) && !HOUR.equals(transformName)) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java index 423472580f0bf6..1e8600115b7e47 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java @@ -500,6 +500,10 @@ private static TColumnType buildPrimitiveColumnType( case DATE: columnType.setType(TPrimitiveType.DATEV2); break; + case TIMESTAMP_NANO: + columnType.setType(((Types.TimestampNanoType) type).shouldAdjustToUTC() + ? TPrimitiveType.STRING : TPrimitiveType.TIMESTAMP_NS); + break; case TIMESTAMP: boolean timestampTz = ((Types.TimestampType) type).shouldAdjustToUTC(); columnType.setType(timestampTz ? TPrimitiveType.TIMESTAMPTZ : TPrimitiveType.DATETIMEV2); @@ -521,8 +525,8 @@ static String serializeInitialDefault(Type type, Object value, boolean enableTim return SingleValueParser.toJson(type, value); } String humanValue = Transforms.identity(type).toHumanString(type, value); - if (type.typeId() == TypeID.TIMESTAMP) { - // Iceberg prints ISO-8601 (2024-01-01T00:00:00); Doris DATETIMEV2 needs a space separator. + if (type.typeId() == TypeID.TIMESTAMP || type.typeId() == TypeID.TIMESTAMP_NANO) { + // Iceberg prints ISO-8601 (2024-01-01T00:00:00); Doris temporal literals need a space separator. String dorisValue = humanValue.replace('T', ' '); // Preserve the offset through the FE-to-BE transport independently of legacy flags. return dorisValue; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java index c1f347470209d4..7192ae2671d663 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java @@ -150,12 +150,16 @@ private static ConnectorType fromPrimitive(Type.PrimitiveType primitive, return ConnectorType.of("TIMESTAMPTZ", ICEBERG_DATETIME_SCALE_MS, 0); } return ConnectorType.of("DATETIMEV2", ICEBERG_DATETIME_SCALE_MS, 0); + case TIMESTAMP_NANO: + // TIMESTAMP_NS is a wall-clock type; zoned nanoseconds need a distinct carrier. + return ConnectorType.of(((Types.TimestampNanoType) primitive).shouldAdjustToUTC() + ? "UNSUPPORTED" : "TIMESTAMP_NS"); case TIME: // iceberg TIME has no Doris analogue -> UNSUPPORTED (explicit, byte-parity with legacy // IcebergUtils which also mapped TIME to Type.UNSUPPORTED). return ConnectorType.of("UNSUPPORTED"); default: - // Any primitive iceberg type Doris cannot represent — notably the v3 types TIMESTAMP_NANO / + // Any primitive iceberg type Doris cannot represent — notably the v3 types // GEOMETRY / GEOGRAPHY / UNKNOWN — degrades to UNSUPPORTED: the table still LOADS and only this // column is present-but-unqueryable. This DIVERGES from legacy (IcebergUtils.icebergPrimitiveType- // ToDorisType threw IllegalArgumentException "Cannot transform unknown type", failing the whole @@ -218,6 +222,8 @@ static Type toIcebergPrimitive(ConnectorType type) { // Legacy parity: timestamp WITHOUT zone (datetime scale dropped — Iceberg timestamps are // microsecond). DorisTypeToIcebergType.atomic returns TimestampType.withoutZone(). return Types.TimestampType.withoutZone(); + case "TIMESTAMP_NS": + return Types.TimestampNanoType.withoutZone(); case "TIMESTAMPTZ": return Types.TimestampType.withZone(); default: diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java new file mode 100644 index 00000000000000..2d24fc96e16ce9 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.spi.ConnectorType; + +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.ZoneId; +import java.util.Collections; + +public class IcebergTimestampNsTest { + @Test + public void nanosecondCalendarPartitionsDoNotBuildOutOfRangeMvBoundaries() { + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); + + for (PartitionSpec spec : new PartitionSpec[] { + PartitionSpec.builderFor(schema).year("ts").build(), + PartitionSpec.builderFor(schema).month("ts").build(), + PartitionSpec.builderFor(schema).day("ts").build(), + PartitionSpec.builderFor(schema).hour("ts").build()}) { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db")); + Table table = catalog.createTable(TableIdentifier.of("db", "t"), schema, spec, + Collections.singletonMap("format-version", "3")); + Assertions.assertFalse(IcebergPartitionUtils.isValidRelatedTable(table)); + } + } + + @Test + public void nanosecondTimestampRoundTripsWithoutTimezone() { + ConnectorType mapped = IcebergTypeMapping.fromIcebergType( + Types.TimestampNanoType.withoutZone(), false, false); + Assertions.assertEquals("TIMESTAMP_NS", mapped.getTypeName()); + Assertions.assertEquals(Types.TimestampNanoType.withoutZone(), + IcebergTypeMapping.toIcebergPrimitive(ConnectorType.of("TIMESTAMP_NS"))); + Assertions.assertEquals("UNSUPPORTED", IcebergTypeMapping.fromIcebergType( + Types.TimestampNanoType.withZone(), false, false).getTypeName()); + } + + @Test + public void identityPartitionsPreserveNanosecondsAcrossEpochAndTimezones() { + for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) { + for (long nanos : new long[] {Long.MIN_VALUE, -1, 0, 1, 123456789, Long.MAX_VALUE}) { + String text = IcebergPartitionUtils.serializePartitionValue( + Types.TimestampNanoType.withoutZone(), nanos, ZoneId.of(zone)); + Assertions.assertEquals(nanos, IcebergPartitionUtils.parsePartitionValueFromString( + text, Types.TimestampNanoType.withoutZone(), ZoneId.of(zone))); + } + } + Assertions.assertThrows(IllegalArgumentException.class, () -> + IcebergPartitionUtils.parsePartitionValueFromString("2262-04-11 23:47:16.854775808", + Types.TimestampNanoType.withoutZone(), ZoneId.of("UTC"))); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java index 257dac6c9718da..75b8ced07c3ff6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java @@ -88,14 +88,14 @@ public void flagIndependentPrimitivesMatchLegacy() { @Test public void unknownAndV3TypesDegradeToUnsupportedByDesign() { // WHY (user decision 2026-07-13, DV-051): iceberg primitive types Doris cannot represent — - // TIMESTAMP_NANO / GEOMETRY / GEOGRAPHY / UNKNOWN — must map to UNSUPPORTED WITHOUT throwing, + // Zoned TIMESTAMP_NANO / GEOMETRY / GEOGRAPHY / UNKNOWN — must map to UNSUPPORTED WITHOUT throwing, // so the table still loads and only the exotic column is // present-but-unqueryable. This deliberately DIVERGES from legacy fe-core, which threw // IllegalArgumentException("Cannot transform unknown type") at schema-load and failed the whole table. // This test PINS the graceful-degradation choice: MUTATION making either default arm throw -> red, // surfacing that the accepted deviation was reverted. (The write direction toIcebergPrimitive still // throws — see toIcebergPrimitiveRejectsUnrepresentableTypes if present / the connector's CREATE path.) - Assertions.assertEquals("UNSUPPORTED", mapOff(Types.TimestampNanoType.withoutZone()).getTypeName()); + Assertions.assertEquals("TIMESTAMP_NS", mapOff(Types.TimestampNanoType.withoutZone()).getTypeName()); Assertions.assertEquals("UNSUPPORTED", mapOff(Types.TimestampNanoType.withZone()).getTypeName()); Assertions.assertEquals("UNSUPPORTED", mapOff(Types.GeometryType.crs84()).getTypeName()); Assertions.assertEquals("UNSUPPORTED", mapOff(Types.GeographyType.crs84()).getTypeName()); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 2d06c4b826d810..4d494696f46358 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -952,6 +952,10 @@ && hasTrustworthyLimitAccounting(table) // Schema IDs are branch-local, even when both branches expose the same current row type. Map> legacyOrcTimestampSchemas = new java.util.IdentityHashMap<>(); Set readFieldIds = scanReadFieldIds(rowType, columns, filter); + // Native INT96/ORC materialization currently narrows these values to microseconds. + // JNI retains the source precision, including nested and filter-only timestamp fields. + boolean needsNanosecondReader = rowType.getFields().stream() + .anyMatch(field -> readFieldIds.contains(field.id()) && containsNanosecondTimestamp(field.type())); // $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered. java.util.function.Supplier legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize( () -> resolveSchemaDictTable(table, paimonHandle)); @@ -971,7 +975,7 @@ && hasTrustworthyLimitAccounting(table) Optional> optRawFiles = dataSplit.convertToRawFiles(); Optional> optDeletionFiles = dataSplit.deletionFiles(); - boolean nativeEligible = shouldUseNativeReader(paimonHandle.isForceJni(), + boolean nativeEligible = shouldUseNativeReader(paimonHandle.isForceJni() || needsNanosecondReader, isForceJniScannerEnabled(session), hasVariantProjection, physicalVariantSchemaIds, optRawFiles); boolean legacyOrcTimestamp = false; if (nativeEligible) { @@ -1935,6 +1939,25 @@ && containsVariant(field.type()))) { return physicalVariantSchemaIds; } + static boolean containsNanosecondTimestamp(DataType type) { + if (type instanceof org.apache.paimon.types.TimestampType) { + return ((org.apache.paimon.types.TimestampType) type).getPrecision() > 6; + } + switch (type.getTypeRoot()) { + case ARRAY: + return containsNanosecondTimestamp(((ArrayType) type).getElementType()); + case MAP: + MapType map = (MapType) type; + return containsNanosecondTimestamp(map.getKeyType()) + || containsNanosecondTimestamp(map.getValueType()); + case ROW: + return ((RowType) type).getFields().stream() + .anyMatch(field -> containsNanosecondTimestamp(field.type())); + default: + return false; + } + } + private static boolean containsVariant(DataType type) { switch (type.getTypeRoot()) { case VARIANT: diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java index a26d4468e87857..b93b70cecde274 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java @@ -156,6 +156,10 @@ private static ConnectorType toDecimalType(DecimalType type) { private static ConnectorType toTimestampType(DataType dataType) { int scale = DEFAULT_TIMESTAMP_SCALE; if (dataType instanceof TimestampType) { + // Keep all source fractional digits instead of truncating high-precision wall-clock values. + if (((TimestampType) dataType).getPrecision() > MAX_TIMESTAMP_SCALE) { + return ConnectorType.of("TIMESTAMP_NS"); + } scale = Math.min(((TimestampType) dataType).getPrecision(), MAX_TIMESTAMP_SCALE); } else if (dataType instanceof LocalZonedTimestampType) { scale = Math.min(((LocalZonedTimestampType) dataType).getPrecision(), MAX_TIMESTAMP_SCALE); @@ -260,6 +264,8 @@ public static DataType toPaimonType(ConnectorType type) { case "DATETIMEV2": // Paimon interprets numeric timestamp strings in units derived from this precision. return new TimestampType("DATETIMEV2".equals(name) ? type.getPrecision() : 0); + case "TIMESTAMP_NS": + return new TimestampType(9); case "TIMESTAMPTZ": // Preserve the instant contract when deriving an external schema. return new LocalZonedTimestampType(type.getPrecision()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataColumnEvolutionTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataColumnEvolutionTest.java index c2fa7337e31c0a..057cbc35edca84 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataColumnEvolutionTest.java @@ -142,9 +142,8 @@ public void modifyColumnCarriesTypeCommentDefaultAndPosition() { } @Test - public void modifyColumnKeepsRemoteTimestampPrecisionBehindLossyProjection() { - // Doris shows a Paimon TIMESTAMP(9) as DATETIMEV2(6). A MODIFY COLUMN that keeps the projected - // type (here it only changes the comment) must not narrow the remote column to TIMESTAMP(6). + public void modifyColumnKeepsRemoteNanosecondTimestampPrecision() { + // A comment-only modification must preserve the remote nanosecond precision. List fields = Arrays.asList( new DataField(0, "id", new IntType(false)), new DataField(1, "ts", new TimestampType(9))); @@ -161,7 +160,7 @@ public void modifyColumnKeepsRemoteTimestampPrecisionBehindLossyProjection() { tsHandle.setPaimonTable(table); tsMetadata.modifyColumn(null, tsHandle, new ConnectorColumn( - "ts", ConnectorType.of("DATETIMEV2", 6, 0), "event time", true, null), null); + "ts", ConnectorType.of("TIMESTAMP_NS"), "event time", true, null), null); Assertions.assertEquals(1, tsOps.lastSchemaChanges.size()); Assertions.assertInstanceOf(SchemaChange.UpdateColumnComment.class, tsOps.lastSchemaChanges.get(0), diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampMappingTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampMappingTest.java index 2eef15ae587dc8..bc0dc0df91bbcf 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampMappingTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampMappingTest.java @@ -26,9 +26,10 @@ class PaimonTimestampMappingTest { @Test void legacyFlagsCannotChangeTimestampSemantics() { for (boolean flag : new boolean[] {false, true}) { - for (int precision : new int[] {0, 3, 6, 9}) { + for (int precision : new int[] {0, 3, 6, 7, 8, 9}) { PaimonTypeMapping.Options options = new PaimonTypeMapping.Options(flag, flag); - Assertions.assertEquals(ConnectorType.of("DATETIMEV2", Math.min(precision, 6), 0), + Assertions.assertEquals(precision > 6 ? ConnectorType.of("TIMESTAMP_NS") + : ConnectorType.of("DATETIMEV2", precision, 0), PaimonTypeMapping.toConnectorType(new org.apache.paimon.types.TimestampType(precision), options)); Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", Math.min(precision, 6), 0), PaimonTypeMapping.toConnectorType( diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampNsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampNsTest.java new file mode 100644 index 00000000000000..e2782752e1e9e6 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTimestampNsTest.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.spi.ConnectorType; + +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PaimonTimestampNsTest { + @Test + public void nanosecondTimestampRoundTripsWithoutTimezone() { + ConnectorType mapped = PaimonTypeMapping.toConnectorType( + new TimestampType(9), PaimonTypeMapping.Options.DEFAULT); + Assertions.assertEquals("TIMESTAMP_NS", mapped.getTypeName()); + Assertions.assertEquals(new TimestampType(9), + PaimonTypeMapping.toPaimonType(ConnectorType.of("TIMESTAMP_NS"))); + Assertions.assertEquals("DATETIMEV2", PaimonTypeMapping.toConnectorType( + new TimestampType(6), PaimonTypeMapping.Options.DEFAULT).getTypeName()); + Assertions.assertEquals("TIMESTAMPTZ", PaimonTypeMapping.toConnectorType( + new LocalZonedTimestampType(9), PaimonTypeMapping.Options.DEFAULT).getTypeName()); + } + + @Test + public void nestedNanosecondsRequireTheLosslessReader() { + Assertions.assertTrue(PaimonScanPlanProvider.containsNanosecondTimestamp(new TimestampType(9))); + Assertions.assertTrue(PaimonScanPlanProvider.containsNanosecondTimestamp( + new ArrayType(new TimestampType(8)))); + Assertions.assertTrue(PaimonScanPlanProvider.containsNanosecondTimestamp( + new MapType(new IntType(), new TimestampType(7)))); + Assertions.assertTrue(PaimonScanPlanProvider.containsNanosecondTimestamp( + RowType.of(new IntType(), new ArrayType(new TimestampType(9))))); + Assertions.assertFalse(PaimonScanPlanProvider.containsNanosecondTimestamp( + new ArrayType(new TimestampType(6)))); + Assertions.assertFalse(PaimonScanPlanProvider.containsNanosecondTimestamp( + new LocalZonedTimestampType(9))); + } +} diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out new file mode 100644 index 00000000000000..8ff8d8715dad7c --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -0,0 +1,407 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !parquet_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !parquet_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !parquet_false_UTC_eq -- +1 + +-- !parquet_false_UTC_range -- +2 +3 +4 +6 + +-- !parquet_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !parquet_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !parquet_false_Asia_Shanghai_eq -- +1 + +-- !parquet_false_Asia_Shanghai_range -- +2 +3 +4 +6 + +-- !parquet_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !parquet_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !parquet_true_UTC_eq -- +1 + +-- !parquet_true_UTC_range -- +2 +3 +4 +6 + +-- !parquet_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !parquet_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !parquet_true_Asia_Shanghai_eq -- +1 + +-- !parquet_true_Asia_Shanghai_range -- +2 +3 +4 +6 + +-- !parquet_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !parquet_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !parquet_identity -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_identity_overwrite -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_year -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_year_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_month -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_month_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_day -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_day_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_hour -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_hour_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_bucket -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !parquet_bucket_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !orc_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !orc_false_UTC_eq -- +1 + +-- !orc_false_UTC_range -- +2 +3 +4 +6 + +-- !orc_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !orc_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !orc_false_Asia_Shanghai_eq -- +1 + +-- !orc_false_Asia_Shanghai_range -- +2 +3 +4 +6 + +-- !orc_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !orc_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !orc_true_UTC_eq -- +1 + +-- !orc_true_UTC_range -- +2 +3 +4 +6 + +-- !orc_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !orc_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !orc_true_Asia_Shanghai_eq -- +1 + +-- !orc_true_Asia_Shanghai_range -- +2 +3 +4 +6 + +-- !orc_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !orc_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !orc_identity -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_identity_overwrite -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_year -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_year_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_month -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_month_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_day -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_day_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_hour -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_hour_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_bucket -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !orc_bucket_overwrite -- +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + diff --git a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out index f4734e460f64e4..c86454f6243eaf 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out +++ b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out @@ -1,15 +1,14 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !c1 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !c2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !ts9_eq_orc -- 1 -- !ts9_eq_parquet -- -1 -- !ts9_lt_orc -- @@ -17,12 +16,12 @@ 1 -- !ts9_gt_parquet -- +1 -- !ts9_ge_parquet -- 1 -- !ts9_in_orc -- -1 -- !ts9_ne_parquet -- 1 @@ -70,16 +69,15 @@ 2024-01-02 10:12:34.123456+08:00 -- !c1 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !c2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !ts9_eq_orc -- 1 -- !ts9_eq_parquet -- -1 -- !ts9_lt_orc -- @@ -87,12 +85,12 @@ 1 -- !ts9_gt_parquet -- +1 -- !ts9_ge_parquet -- 1 -- !ts9_in_orc -- -1 -- !ts9_ne_parquet -- 1 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out new file mode 100644 index 00000000000000..7d4921ac9809e0 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out @@ -0,0 +1,659 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ns_parquet_false_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !ns_parquet_false_false_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_false_false_UTC_eq -- +1 + +-- !ns_parquet_false_false_false_UTC_nested -- +1 + +-- !ns_parquet_false_false_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_false_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_false_false_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_false_false_false_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_false_false_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_false_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_false_true_UTC_eq -- +1 + +-- !ns_parquet_false_false_true_UTC_nested -- +1 + +-- !ns_parquet_false_false_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_false_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_false_true_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_false_false_true_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_false_false_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_true_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_true_false_UTC_eq -- +1 + +-- !ns_parquet_false_true_false_UTC_nested -- +1 + +-- !ns_parquet_false_true_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_true_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_true_false_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_false_true_false_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_false_true_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_true_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_true_true_UTC_eq -- +1 + +-- !ns_parquet_false_true_true_UTC_nested -- +1 + +-- !ns_parquet_false_true_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_true_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_false_true_true_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_false_true_true_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_false_true_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_false_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !ns_parquet_true_false_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_false_false_UTC_eq -- +1 + +-- !ns_parquet_true_false_false_UTC_nested -- +1 + +-- !ns_parquet_true_false_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_false_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_false_false_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_true_false_false_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_true_false_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_false_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_false_true_UTC_eq -- +1 + +-- !ns_parquet_true_false_true_UTC_nested -- +1 + +-- !ns_parquet_true_false_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_false_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_false_true_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_true_false_true_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_true_false_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_true_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_true_false_UTC_eq -- +1 + +-- !ns_parquet_true_true_false_UTC_nested -- +1 + +-- !ns_parquet_true_true_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_true_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_true_false_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_true_true_false_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_true_true_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_true_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_true_true_UTC_eq -- +1 + +-- !ns_parquet_true_true_true_UTC_nested -- +1 + +-- !ns_parquet_true_true_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_true_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_true_true_Asia_Shanghai_eq -- +1 + +-- !ns_parquet_true_true_true_Asia_Shanghai_nested -- +1 + +-- !ns_parquet_true_true_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_parquet_true_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_parquet_true_upsert -- +1 2024-01-02 03:04:05.123456790 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + +-- !ns_orc_false_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !ns_orc_false_false_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_false_false_UTC_eq -- +1 + +-- !ns_orc_false_false_false_UTC_nested -- +1 + +-- !ns_orc_false_false_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_false_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_false_false_Asia_Shanghai_eq -- +1 + +-- !ns_orc_false_false_false_Asia_Shanghai_nested -- +1 + +-- !ns_orc_false_false_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_false_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_false_true_UTC_eq -- +1 + +-- !ns_orc_false_false_true_UTC_nested -- +1 + +-- !ns_orc_false_false_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_false_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_false_true_Asia_Shanghai_eq -- +1 + +-- !ns_orc_false_false_true_Asia_Shanghai_nested -- +1 + +-- !ns_orc_false_false_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_true_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_true_false_UTC_eq -- +1 + +-- !ns_orc_false_true_false_UTC_nested -- +1 + +-- !ns_orc_false_true_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_true_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_true_false_Asia_Shanghai_eq -- +1 + +-- !ns_orc_false_true_false_Asia_Shanghai_nested -- +1 + +-- !ns_orc_false_true_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_true_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_true_true_UTC_eq -- +1 + +-- !ns_orc_false_true_true_UTC_nested -- +1 + +-- !ns_orc_false_true_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_true_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_false_true_true_Asia_Shanghai_eq -- +1 + +-- !ns_orc_false_true_true_Asia_Shanghai_nested -- +1 + +-- !ns_orc_false_true_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_false_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_schema -- +id int Yes true \N +ts timestamp_ns Yes true \N +items array Yes true \N +detail struct Yes true \N +entries map Yes true \N + +-- !ns_orc_true_false_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_false_false_UTC_eq -- +1 + +-- !ns_orc_true_false_false_UTC_nested -- +1 + +-- !ns_orc_true_false_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_false_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_false_false_Asia_Shanghai_eq -- +1 + +-- !ns_orc_true_false_false_Asia_Shanghai_nested -- +1 + +-- !ns_orc_true_false_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_false_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_false_true_UTC_eq -- +1 + +-- !ns_orc_true_false_true_UTC_nested -- +1 + +-- !ns_orc_true_false_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_false_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_false_true_Asia_Shanghai_eq -- +1 + +-- !ns_orc_true_false_true_Asia_Shanghai_nested -- +1 + +-- !ns_orc_true_false_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_true_false_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_true_false_UTC_eq -- +1 + +-- !ns_orc_true_true_false_UTC_nested -- +1 + +-- !ns_orc_true_true_false_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_true_false_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_true_false_Asia_Shanghai_eq -- +1 + +-- !ns_orc_true_true_false_Asia_Shanghai_nested -- +1 + +-- !ns_orc_true_true_false_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_true_true_UTC_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_true_true_UTC_eq -- +1 + +-- !ns_orc_true_true_true_UTC_nested -- +1 + +-- !ns_orc_true_true_true_UTC_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_true_true_Asia_Shanghai_values -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_true_true_Asia_Shanghai_eq -- +1 + +-- !ns_orc_true_true_true_Asia_Shanghai_nested -- +1 + +-- !ns_orc_true_true_true_Asia_Shanghai_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 + +-- !ns_orc_true_copy -- +1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} +2 2024-01-02 03:04:05.123456788 [] \N {} +3 1969-12-31 23:59:58.999999999 \N \N \N +4 1970-01-01 00:00:00.000000001 \N \N \N +5 \N \N \N \N +6 1677-09-21 00:12:43.145224192 \N \N \N +7 2262-04-11 23:47:16.854775807 \N \N \N + +-- !ns_orc_true_upsert -- +1 2024-01-02 03:04:05.123456790 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy new file mode 100644 index 00000000000000..af142fd7856743 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { + if (!context.config.otherConfigs.get("enableIcebergTest")?.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalog = "test_iceberg_timestamp_ns" + def zone = sql("SELECT @@time_zone")[0][0] + def scanner = sql("SHOW VARIABLES LIKE 'enable_file_scanner_v2'")[0][1] + try { + sql "DROP CATALOG IF EXISTS ${catalog}" + sql """CREATE CATALOG ${catalog} PROPERTIES ( + "type"="iceberg", "iceberg.catalog.type"="rest", "uri"="http://${host}:${restPort}", + "s3.endpoint"="http://${host}:${minioPort}", "s3.access_key"="admin", + "s3.secret_key"="password", "s3.region"="us-east-1", "use_path_style"="true")""" + sql "SWITCH ${catalog}" + sql "CREATE DATABASE IF NOT EXISTS timestamp_ns_roundtrip" + sql "USE timestamp_ns_roundtrip" + for (String format : ["parquet", "orc"]) { + String table = "ns_${format}" + sql "DROP TABLE IF EXISTS ${table}" + sql """CREATE TABLE ${table} (id INT, ts TIMESTAMP_NS, items ARRAY, + detail STRUCT, entries MAP) + PROPERTIES ("format-version"="3", "write.format.default"="${format}")""" + sql "SET time_zone='Asia/Shanghai'" + sql """INSERT INTO ${table} VALUES + (1, '2024-01-02 03:04:05.123456789', ['2024-01-02 03:04:05.123456789', NULL], + named_struct('t', CAST('1969-12-31 23:59:58.999999999' AS TIMESTAMP_NS)), + map('x', CAST('1970-01-01 00:00:00.000000001' AS TIMESTAMP_NS))), + (2, '2024-01-02 03:04:05.123456788', [], NULL, map()), + (3, '1969-12-31 23:59:58.999999999', NULL, NULL, NULL), + (4, '1970-01-01 00:00:00.000000001', NULL, NULL, NULL), + (5, NULL, NULL, NULL, NULL), + (6, '1677-09-21 00:12:43.145224192', NULL, NULL, NULL), + (7, '2262-04-11 23:47:16.854775807', NULL, NULL, NULL)""" + "qt_${format}_schema" "DESC ${table}" + for (boolean v2 : [false, true]) { + sql "SET enable_file_scanner_v2=${v2}" + for (String sessionZone : ["UTC", "Asia/Shanghai"]) { + sql "SET time_zone='${sessionZone}'" + String tag = "${format}_${v2}_${sessionZone.replace('/', '_')}" + "qt_${tag}_values" "SELECT * FROM ${table} ORDER BY id" + // The two positive timestamps differ only in the last nanosecond. + "qt_${tag}_eq" """SELECT id FROM ${table} + WHERE ts=CAST('2024-01-02 03:04:05.123456789' AS TIMESTAMP_NS) ORDER BY id""" + "qt_${tag}_range" """SELECT id FROM ${table} + WHERE ts + def test_scale = { -> def ts_scale_orc = """select * from ts_scale_orc""" def ts_scale_parquet = """select * from ts_scale_parquet""" qt_c1 ts_scale_orc qt_c2 ts_scale_parquet - // JNI truncates ORC nanoseconds while native ORC rounds; predicates must use each - // reader's visible microsecond value to keep pruning and decoding consistent. Keep the - // predicates below on ts9 so file pruning must retain the row whose source value has - // nanoseconds beyond the Doris-visible literal. + // Preserve all nine digits for both formats and scanner selections. order_qt_ts9_eq_orc """ select id from ts_scale_orc - where ts9 = '${orcTimestamp}' + where ts9 = '2024-01-02 10:04:05.123456789' order by id """ order_qt_ts9_eq_parquet """ @@ -117,12 +114,12 @@ suite("paimon_timestamp_types", "p0,external") { """ order_qt_ts9_lt_orc """ select id from ts_scale_orc - where ts9 < '${orcTimestamp}' + where ts9 < '2024-01-02 10:04:05.123456789' order by id """ order_qt_ts9_le_orc """ select id from ts_scale_orc - where ts9 <= '${orcTimestamp}' + where ts9 <= '2024-01-02 10:04:05.123456789' order by id """ order_qt_ts9_gt_parquet """ @@ -149,19 +146,16 @@ suite("paimon_timestamp_types", "p0,external") { } sql """set force_jni_scanner=true""" - test_scale("2024-01-02 10:04:05.123456") + test_scale() // test_ltz_ntz("test_timestamp_ntz_ltz_orc") // test_ltz_ntz("test_timestamp_ntz_ltz_parquet") test_ltz_ntz_simple("test_timestamp_ntz_ltz_simple_orc", "2024-01-02 10:12:34.123456+08:00") // test_ltz_ntz_simple("test_timestamp_ntz_ltz_simple_parquet") - // Native ORC rounds scales 7-9 to microseconds; JNI truncates. Native Parquet uses - // Paimon history semantics for high-precision INT96 instead of the session timezone. + // Automatic reader selection must preserve the same nanosecond values as forced JNI. sql """set force_jni_scanner=false""" - // Precision-aware native predicates use V2; V1 compares raw nanoseconds before truncation. - // SELECT * still falls back to JNI because it includes legacy ORC LTZ fields. sql """set enable_file_scanner_v2=true""" - test_scale("2024-01-02 10:04:05.123457") + test_scale() // Keep testing legacy LTZ fallback with V2 disabled independently of the NTZ predicate tests. sql """set enable_file_scanner_v2=false""" // V1 must still decode every timestamp scale. Legacy ORC LTZ files need Paimon's diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy new file mode 100644 index 00000000000000..1c9acb0bdd685c --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_timestamp_ns", "p0,external,paimon") { + if (!context.config.otherConfigs.get("enablePaimonTest")?.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalog = "test_paimon_timestamp_ns" + def zone = sql("SELECT @@time_zone")[0][0] + def scanner = sql("SHOW VARIABLES LIKE 'enable_file_scanner_v2'")[0][1] + def jni = sql("SHOW VARIABLES LIKE 'force_jni_scanner'")[0][1] + try { + sql "DROP CATALOG IF EXISTS ${catalog}" + sql """CREATE CATALOG ${catalog} PROPERTIES ( + "type"="paimon", "paimon.catalog.type"="filesystem", "warehouse"="s3://warehouse/wh", + "s3.endpoint"="http://${host}:${minioPort}", "s3.access_key"="admin", + "s3.secret_key"="password", "s3.region"="us-east-1", "s3.path.style.access"="true")""" + sql "SWITCH ${catalog}" + sql "CREATE DATABASE IF NOT EXISTS timestamp_ns_roundtrip" + sql "USE timestamp_ns_roundtrip" + for (String format : ["parquet", "orc"]) { + for (boolean primaryKey : [false, true]) { + String table = "ns_${format}_${primaryKey}" + String key = primaryKey ? ', "primary-key"="id", "bucket"="1"' : ', "bucket"="-1"' + sql "DROP TABLE IF EXISTS ${table}" + sql """CREATE TABLE ${table} (id INT NOT NULL, ts TIMESTAMP_NS, + items ARRAY, detail STRUCT, entries MAP) + ENGINE=paimon PROPERTIES ("file.format"="${format}" ${key})""" + sql "SET time_zone='Asia/Shanghai'" + sql """INSERT INTO ${table} VALUES + (1, '2024-01-02 03:04:05.123456789', ['2024-01-02 03:04:05.123456789', NULL], + named_struct('t', CAST('1969-12-31 23:59:58.999999999' AS TIMESTAMP_NS)), + map('x', CAST('1970-01-01 00:00:00.000000001' AS TIMESTAMP_NS))), + (2, '2024-01-02 03:04:05.123456788', [], NULL, map()), + (3, '1969-12-31 23:59:58.999999999', NULL, NULL, NULL), + (4, '1970-01-01 00:00:00.000000001', NULL, NULL, NULL), + (5, NULL, NULL, NULL, NULL), + (6, '1677-09-21 00:12:43.145224192', NULL, NULL, NULL), + (7, '2262-04-11 23:47:16.854775807', NULL, NULL, NULL)""" + "qt_${table}_schema" "DESC ${table}" + for (boolean v2 : [false, true]) { + sql "SET enable_file_scanner_v2=${v2}" + for (boolean forceJni : [false, true]) { + sql "SET force_jni_scanner=${forceJni}" + for (String sessionZone : ["UTC", "Asia/Shanghai"]) { + sql "SET time_zone='${sessionZone}'" + String tag = "${table}_${v2}_${forceJni}_${sessionZone.replace('/', '_')}" + "qt_${tag}_values" "SELECT * FROM ${table} ORDER BY id" + // Filter-only high-precision columns also require lossless materialization. + "qt_${tag}_eq" """SELECT id FROM ${table} + WHERE ts=CAST('2024-01-02 03:04:05.123456789' AS TIMESTAMP_NS) ORDER BY id""" + "qt_${tag}_nested" """SELECT id FROM ${table} WHERE + items[1]=CAST('2024-01-02 03:04:05.123456789' AS TIMESTAMP_NS) ORDER BY id""" + "qt_${tag}_aggregate" "SELECT min(ts), max(ts), count(DISTINCT ts) FROM ${table}" + } + } + } + sql "DROP TABLE IF EXISTS ${table}_copy" + sql """CREATE TABLE ${table}_copy PROPERTIES ("file.format"="${format}") + AS SELECT * FROM ${table}""" + "qt_${table}_copy" "SELECT * FROM ${table}_copy ORDER BY id" + if (primaryKey) { + sql """INSERT INTO ${table} VALUES (1, '2024-01-02 03:04:05.123456790', NULL, NULL, NULL)""" + "qt_${table}_upsert" "SELECT id, ts FROM ${table} ORDER BY id" + } + } + } + } finally { + sql "SET time_zone='${zone}'" + sql "SET enable_file_scanner_v2=${scanner}" + sql "SET force_jni_scanner=${jni}" + sql "SWITCH internal" + } +} From c2ea4eb0fd6a0efb3c2827d26df879516090acb5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 12 Oct 2026 13:30:13 +0800 Subject: [PATCH 02/13] [refactor](datalake) Limit native nanosecond reads to format V2 ### What problem does this PR solve? Problem Summary: Keep format V1 readers unchanged when adding external nanosecond timestamp support. Restore legacy ORC and Parquet reader code, restrict the Iceberg regression to format V2, and regenerate its results. Iceberg writers and Paimon JNI read/write support are retained. ### Release note Iceberg nanosecond reads require enable_file_scanner_v2=true. ### Check List (For Author) - Test: ASAN BE build; 117 BE unit tests; Iceberg and Paimon nanosecond regression suites; clang-format 16; clang-tidy; header hygiene. - Behavior changed: Yes, nanosecond native reads are supported by format V2. - Does this need documentation: Yes, external type-mapping documentation. --- be/src/format/orc/vorc_reader.cpp | 13 -- .../format/parquet/parquet_column_convert.cpp | 2 - .../format/parquet/parquet_column_convert.h | 13 -- be/src/format/parquet/schema_desc.cpp | 5 - .../iceberg/test_iceberg_timestamp_ns.out | 116 +++--------------- .../iceberg/test_iceberg_timestamp_ns.groovy | 24 ++-- 6 files changed, 27 insertions(+), 146 deletions(-) diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index c6544cb178a0dd..958ed0f3ca96a2 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -29,7 +29,6 @@ #include #include -#include "core/data_type_serde/orc_serde_utils.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vexpr.h" @@ -419,12 +418,6 @@ Status OrcReader::_create_file_reader() { try { _reader = orc::createReader( std::unique_ptr(_file_input_stream.release()), options); - if (orc_serde_utils::has_nanosecond_timestamp(_reader->getType())) { - // ORC and cctz disagree on pre-transition local offsets. UTC preserves the - // wall-clock value across the full nanosecond range without that conversion. - _ctz = "GMT"; - _time_zone = cctz::utc_time_zone(); - } } catch (std::exception& e) { // invoker maybe just skip Status.NotFound and continue // so we need distinguish between it and other kinds of errors @@ -1605,10 +1598,6 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); } case orc::TypeKind::TIMESTAMP: - if (orc_type->hasAttributeKey("iceberg.timestamp-unit") && - orc_type->getAttributeValue("iceberg.timestamp-unit") == "NANOS") { - return DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, true); - } return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, true, 0, 6); case orc::TypeKind::DECIMAL: @@ -1959,14 +1948,12 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, #undef DISPATCH case PrimitiveType::TYPE_INT: return _decode_int32_column(col_name, data_column, cvb, num_values); - case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_UUID: { OrcDecodedColumnView view; view.file_type = orc_column_type; view.selected_type = orc_column_type; view.batch = cvb; view.rows = num_values; - view.timezone = &_time_zone; return data_type->get_serde()->read_column_from_orc(*data_column, view); } case PrimitiveType::TYPE_DECIMAL32: diff --git a/be/src/format/parquet/parquet_column_convert.cpp b/be/src/format/parquet/parquet_column_convert.cpp index 65f69bc9aca8c0..c0fbadbc6d56da 100644 --- a/be/src/format/parquet/parquet_column_convert.cpp +++ b/be/src/format/parquet/parquet_column_convert.cpp @@ -308,8 +308,6 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv convert_params.get(), physical_converter); } else if (src_logical_primitive == TYPE_DATEV2) { physical_converter = std::make_unique(); - } else if (src_logical_primitive == TYPE_TIMESTAMP_NS) { - physical_converter = std::make_unique(); } else if (src_logical_primitive == TYPE_DATETIMEV2) { if (src_physical_type == tparquet::Type::INT96) { // int96 only stores nanoseconds in standard parquet file diff --git a/be/src/format/parquet/parquet_column_convert.h b/be/src/format/parquet/parquet_column_convert.h index 5a1c8b7260e1b1..083711a5ce34a3 100644 --- a/be/src/format/parquet/parquet_column_convert.h +++ b/be/src/format/parquet/parquet_column_convert.h @@ -836,19 +836,6 @@ class Int32ToDate : public PhysicalToLogicalConverter { } }; -struct Int64ToTimestampNs : public PhysicalToLogicalConverter { - Status physical_convert(ColumnPtr& source, ColumnPtr& target) override { - const auto& input = assert_cast(*remove_nullable(source)).get_data(); - auto& output = - assert_cast(*get_mutable_inner_column(target)).get_data(); - // Only the unadjusted TIMESTAMP(NANOS) annotation selects this converter. - for (int64_t nanos : input) { - output.emplace_back(nanos); - } - return Status::OK(); - } -}; - struct Int64ToTimestamp : public PhysicalToLogicalConverter { Status physical_convert(ColumnPtr& src_physical_col, ColumnPtr& src_logical_column) override { ColumnPtr src_col = remove_nullable(src_physical_col); diff --git a/be/src/format/parquet/schema_desc.cpp b/be/src/format/parquet/schema_desc.cpp index 1f22091df9c8f8..c2c0ed22ed33df 100644 --- a/be/src/format/parquet/schema_desc.cpp +++ b/be/src/format/parquet/schema_desc.cpp @@ -302,11 +302,6 @@ std::pair FieldDescriptor::convert_to_doris_type( } else if (logicalType.__isset.TIME) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); } else if (logicalType.__isset.TIMESTAMP) { - if (logicalType.TIMESTAMP.unit.__isset.NANOS && !logicalType.TIMESTAMP.isAdjustedToUTC) { - // Materialize nanoseconds before any table cast; DATETIMEV2 would discard the remainder. - ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, nullable); - return ans; - } if (_enable_mapping_timestamp_tz) { if (logicalType.TIMESTAMP.isAdjustedToUTC) { // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out index 8ff8d8715dad7c..358ac94738fc0c 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -6,7 +6,7 @@ items array Yes true \N detail struct Yes true \N entries map Yes true \N --- !parquet_false_UTC_values -- +-- !parquet_UTC_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} 3 1969-12-31 23:59:58.999999999 \N \N \N @@ -15,19 +15,19 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N --- !parquet_false_UTC_eq -- +-- !parquet_UTC_eq -- 1 --- !parquet_false_UTC_range -- +-- !parquet_UTC_range -- 2 3 4 6 --- !parquet_false_UTC_aggregate -- +-- !parquet_UTC_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 --- !parquet_false_Asia_Shanghai_values -- +-- !parquet_Asia_Shanghai_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} 3 1969-12-31 23:59:58.999999999 \N \N \N @@ -36,58 +36,16 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N --- !parquet_false_Asia_Shanghai_eq -- +-- !parquet_Asia_Shanghai_eq -- 1 --- !parquet_false_Asia_Shanghai_range -- +-- !parquet_Asia_Shanghai_range -- 2 3 4 6 --- !parquet_false_Asia_Shanghai_aggregate -- -1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 - --- !parquet_true_UTC_values -- -1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} -2 2024-01-02 03:04:05.123456788 [] \N {} -3 1969-12-31 23:59:58.999999999 \N \N \N -4 1970-01-01 00:00:00.000000001 \N \N \N -5 \N \N \N \N -6 1677-09-21 00:12:43.145224192 \N \N \N -7 2262-04-11 23:47:16.854775807 \N \N \N - --- !parquet_true_UTC_eq -- -1 - --- !parquet_true_UTC_range -- -2 -3 -4 -6 - --- !parquet_true_UTC_aggregate -- -1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 - --- !parquet_true_Asia_Shanghai_values -- -1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} -2 2024-01-02 03:04:05.123456788 [] \N {} -3 1969-12-31 23:59:58.999999999 \N \N \N -4 1970-01-01 00:00:00.000000001 \N \N \N -5 \N \N \N \N -6 1677-09-21 00:12:43.145224192 \N \N \N -7 2262-04-11 23:47:16.854775807 \N \N \N - --- !parquet_true_Asia_Shanghai_eq -- -1 - --- !parquet_true_Asia_Shanghai_range -- -2 -3 -4 -6 - --- !parquet_true_Asia_Shanghai_aggregate -- +-- !parquet_Asia_Shanghai_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 -- !parquet_copy -- @@ -209,49 +167,7 @@ items array Yes true \N detail struct Yes true \N entries map Yes true \N --- !orc_false_UTC_values -- -1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} -2 2024-01-02 03:04:05.123456788 [] \N {} -3 1969-12-31 23:59:58.999999999 \N \N \N -4 1970-01-01 00:00:00.000000001 \N \N \N -5 \N \N \N \N -6 1677-09-21 00:12:43.145224192 \N \N \N -7 2262-04-11 23:47:16.854775807 \N \N \N - --- !orc_false_UTC_eq -- -1 - --- !orc_false_UTC_range -- -2 -3 -4 -6 - --- !orc_false_UTC_aggregate -- -1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 - --- !orc_false_Asia_Shanghai_values -- -1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} -2 2024-01-02 03:04:05.123456788 [] \N {} -3 1969-12-31 23:59:58.999999999 \N \N \N -4 1970-01-01 00:00:00.000000001 \N \N \N -5 \N \N \N \N -6 1677-09-21 00:12:43.145224192 \N \N \N -7 2262-04-11 23:47:16.854775807 \N \N \N - --- !orc_false_Asia_Shanghai_eq -- -1 - --- !orc_false_Asia_Shanghai_range -- -2 -3 -4 -6 - --- !orc_false_Asia_Shanghai_aggregate -- -1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 - --- !orc_true_UTC_values -- +-- !orc_UTC_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} 3 1969-12-31 23:59:58.999999999 \N \N \N @@ -260,19 +176,19 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N --- !orc_true_UTC_eq -- +-- !orc_UTC_eq -- 1 --- !orc_true_UTC_range -- +-- !orc_UTC_range -- 2 3 4 6 --- !orc_true_UTC_aggregate -- +-- !orc_UTC_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 --- !orc_true_Asia_Shanghai_values -- +-- !orc_Asia_Shanghai_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} 3 1969-12-31 23:59:58.999999999 \N \N \N @@ -281,16 +197,16 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N --- !orc_true_Asia_Shanghai_eq -- +-- !orc_Asia_Shanghai_eq -- 1 --- !orc_true_Asia_Shanghai_range -- +-- !orc_Asia_Shanghai_range -- 2 3 4 6 --- !orc_true_Asia_Shanghai_aggregate -- +-- !orc_Asia_Shanghai_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 -- !orc_copy -- diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy index af142fd7856743..a89616554ffba0 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -26,6 +26,7 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { def zone = sql("SELECT @@time_zone")[0][0] def scanner = sql("SHOW VARIABLES LIKE 'enable_file_scanner_v2'")[0][1] try { + sql "SET enable_file_scanner_v2=true" sql "DROP CATALOG IF EXISTS ${catalog}" sql """CREATE CATALOG ${catalog} PROPERTIES ( "type"="iceberg", "iceberg.catalog.type"="rest", "uri"="http://${host}:${restPort}", @@ -52,19 +53,16 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { (6, '1677-09-21 00:12:43.145224192', NULL, NULL, NULL), (7, '2262-04-11 23:47:16.854775807', NULL, NULL, NULL)""" "qt_${format}_schema" "DESC ${table}" - for (boolean v2 : [false, true]) { - sql "SET enable_file_scanner_v2=${v2}" - for (String sessionZone : ["UTC", "Asia/Shanghai"]) { - sql "SET time_zone='${sessionZone}'" - String tag = "${format}_${v2}_${sessionZone.replace('/', '_')}" - "qt_${tag}_values" "SELECT * FROM ${table} ORDER BY id" - // The two positive timestamps differ only in the last nanosecond. - "qt_${tag}_eq" """SELECT id FROM ${table} - WHERE ts=CAST('2024-01-02 03:04:05.123456789' AS TIMESTAMP_NS) ORDER BY id""" - "qt_${tag}_range" """SELECT id FROM ${table} - WHERE ts Date: Mon, 12 Oct 2026 14:18:50 +0800 Subject: [PATCH 03/13] [refactor](iceberg) Split timestamp kinds into four distinct types ### What problem does this PR solve? Related PR: #68934 Problem Summary: Represent timestamp, timestamptz, timestamp_ns and timestamptz_ns with independent classes and type IDs instead of two boolean attributes. Update schema parsing, Arrow conversion and recursive Parquet nanosecond detection while preserving their existing units and timezones. ### Release note None ### Check List (For Author) - Test: 471 BE unit tests under ASAN; clang-format 16 and build hygiene checks - Behavior changed: No - Does this need documentation: No --- .../iceberg/viceberg_partition_writer.cpp | 5 +-- .../table/iceberg/arrow_schema_util.cpp | 20 +++++++---- be/src/format/table/iceberg/types.cpp | 8 ++--- be/src/format/table/iceberg/types.h | 36 ++++++++++--------- .../iceberg/viceberg_table_writer_test.cpp | 2 +- .../table/iceberg/arrow_schema_util_test.cpp | 32 ++++++++++++----- be/test/format/table/iceberg/type_test.cpp | 31 +++++++++++++--- 7 files changed, 92 insertions(+), 42 deletions(-) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 104c1e91cb714c..0be4d1d371db2a 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -33,8 +33,9 @@ namespace doris { namespace { bool contains_nanosecond_timestamp(const iceberg::Type& type) { switch (type.type_id()) { - case iceberg::TypeID::TIMESTAMP: - return static_cast(type).is_nanosecond(); + case iceberg::TypeID::TIMESTAMP_NS: + case iceberg::TypeID::TIMESTAMPTZ_NS: + return true; case iceberg::TypeID::STRUCT: for (const auto& field : static_cast(type).fields()) { if (contains_nanosecond_timestamp(*field.field_type())) { diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index 557a813d1ea18f..18aec26888f8e3 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -84,13 +84,21 @@ Status IcebergArrowSchemaConvertor::convert_to_arrow_field( arrow_type = arrow::date32(); break; - case iceberg::TypeID::TIMESTAMP: { - iceberg::TimestampType* t_type = static_cast(field.field_type()); - std::string real_tz = t_type->should_adjust_to_utc() ? _timezone : ""; - arrow_type = std::make_shared( - t_type->is_nanosecond() ? arrow::TimeUnit::NANO : arrow::TimeUnit::MICRO, real_tz); + case iceberg::TypeID::TIMESTAMP: + arrow_type = arrow::timestamp(arrow::TimeUnit::MICRO); + break; + + case iceberg::TypeID::TIMESTAMPTZ: + arrow_type = arrow::timestamp(arrow::TimeUnit::MICRO, _timezone); + break; + + case iceberg::TypeID::TIMESTAMP_NS: + arrow_type = arrow::timestamp(arrow::TimeUnit::NANO); + break; + + case iceberg::TypeID::TIMESTAMPTZ_NS: + arrow_type = arrow::timestamp(arrow::TimeUnit::NANO, _timezone); break; - } case iceberg::TypeID::STRING: arrow_type = arrow::utf8(); diff --git a/be/src/format/table/iceberg/types.cpp b/be/src/format/table/iceberg/types.cpp index cfdfb7479f4050..7046e3b4f6fe58 100644 --- a/be/src/format/table/iceberg/types.cpp +++ b/be/src/format/table/iceberg/types.cpp @@ -161,13 +161,13 @@ std::unique_ptr Types::from_primitive_string(const std::string& t } else if (lower_type_string == "time") { return std::make_unique(); } else if (lower_type_string == "timestamptz") { - return std::make_unique(true); + return std::make_unique(); } else if (lower_type_string == "timestamp") { - return std::make_unique(false); + return std::make_unique(); } else if (lower_type_string == "timestamp_ns") { - return std::make_unique(false, true); + return std::make_unique(); } else if (lower_type_string == "timestamptz_ns") { - return std::make_unique(true, true); + return std::make_unique(); } else if (lower_type_string == "string") { return std::make_unique(); } else if (lower_type_string == "uuid") { diff --git a/be/src/format/table/iceberg/types.h b/be/src/format/table/iceberg/types.h index a4fb844897deee..b4801dd0a5e6f2 100644 --- a/be/src/format/table/iceberg/types.h +++ b/be/src/format/table/iceberg/types.h @@ -41,6 +41,9 @@ enum TypeID { DATE, TIME, TIMESTAMP, + TIMESTAMPTZ, + TIMESTAMP_NS, + TIMESTAMPTZ_NS, STRING, UUID, FIXED, @@ -313,28 +316,29 @@ class StringType : public PrimitiveType { std::string to_string() const override { return "string"; } }; +// Keep unit and timezone semantics in the type identity so dispatch cannot silently lose them. class TimestampType : public PrimitiveType { public: - ~TimestampType() override = default; - - TimestampType(bool adjust_to_utc, bool nanoseconds = false) - : _adjust_to_utc(adjust_to_utc), _nanoseconds(nanoseconds) {} - TypeID type_id() const override { return TypeID::TIMESTAMP; } + std::string to_string() const override { return "timestamp"; } +}; - bool should_adjust_to_utc() const { return _adjust_to_utc; } - bool is_nanosecond() const { return _nanoseconds; } +class TimestampTzType : public PrimitiveType { +public: + TypeID type_id() const override { return TypeID::TIMESTAMPTZ; } + std::string to_string() const override { return "timestamptz"; } +}; - std::string to_string() const override { - if (_nanoseconds) { - return should_adjust_to_utc() ? "timestamptz_ns" : "timestamp_ns"; - } - return should_adjust_to_utc() ? "timestamptz" : "timestamp"; - } +class TimestampNsType : public PrimitiveType { +public: + TypeID type_id() const override { return TypeID::TIMESTAMP_NS; } + std::string to_string() const override { return "timestamp_ns"; } +}; -private: - bool _adjust_to_utc; - bool _nanoseconds; +class TimestampTzNsType : public PrimitiveType { +public: + TypeID type_id() const override { return TypeID::TIMESTAMPTZ_NS; } + std::string to_string() const override { return "timestamptz_ns"; } }; class TimeType : public PrimitiveType { diff --git a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp index ff1159bcd83c6b..21397daa14149d 100644 --- a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp @@ -66,7 +66,7 @@ class RecordingPartitionWriter : public IPartitionWriterBase { TEST(VIcebergTableWriterTest, TimestampIdentityPreservesUtcCommitAndNull) { std::vector fields; - fields.emplace_back(false, 1, "event_time", std::make_unique(true), + fields.emplace_back(false, 1, "event_time", std::make_unique(), std::nullopt); auto schema = std::make_shared(std::move(fields)); auto spec = iceberg::PartitionSpecParser::from_json( diff --git a/be/test/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/format/table/iceberg/arrow_schema_util_test.cpp index cdb3f4fc07cb66..da85ac4b764921 100644 --- a/be/test/format/table/iceberg/arrow_schema_util_test.cpp +++ b/be/test/format/table/iceberg/arrow_schema_util_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -54,15 +55,28 @@ class ArrowSchemaUtilTest : public testing::Test { virtual ~ArrowSchemaUtilTest() = default; }; -TEST(ArrowSchemaUtilTest, TimestampNsSchemaPreservesNanosecondUnit) { - const std::string json = R"({"type":"struct","fields":[ - {"id":1,"name":"ts","required":false,"type":"timestamp_ns"}]})"; - auto schema = SchemaParser::from_json(json); - IcebergArrowSchemaConvertor converter(*schema, "Asia/Shanghai", json); - std::shared_ptr arrow_schema; - ASSERT_TRUE(converter.get_arrow_schema(&arrow_schema).ok()); - EXPECT_TRUE(arrow_schema->field(0)->type()->Equals(arrow::timestamp(arrow::TimeUnit::NANO))); - EXPECT_EQ(schema->columns()[0].field_type()->to_string(), "timestamp_ns"); +TEST(ArrowSchemaUtilTest, TimestampKindsPreserveUnitAndTimezone) { + const char* names[] = {"timestamp", "timestamptz", "timestamp_ns", "timestamptz_ns"}; + const arrow::TimeUnit::type units[] = {arrow::TimeUnit::MICRO, arrow::TimeUnit::MICRO, + arrow::TimeUnit::NANO, arrow::TimeUnit::NANO}; + const char* zones[] = {"", "Asia/Shanghai", "", "Asia/Shanghai"}; + for (int i = 0; i < 4; ++i) { + SCOPED_TRACE(names[i]); + const auto json = fmt::format(R"({{"type":"struct","fields":[ + {{"id":1,"name":"ts","required":false,"type":"{0}"}}, + {{"id":2,"name":"nested","required":false,"type":{{"type":"list", + "element-id":3,"element-required":false,"element":"{0}"}}}}]}})", + names[i]); + auto schema = SchemaParser::from_json(json); + IcebergArrowSchemaConvertor converter(*schema, "Asia/Shanghai", json); + std::shared_ptr arrow_schema; + ASSERT_TRUE(converter.get_arrow_schema(&arrow_schema).ok()); + const auto expected = arrow::timestamp(units[i], zones[i]); + EXPECT_TRUE(arrow_schema->field(0)->type()->Equals(expected)); + auto list_type = std::static_pointer_cast(arrow_schema->field(1)->type()); + EXPECT_TRUE(list_type->value_type()->Equals(expected)); + EXPECT_EQ(schema->columns()[0].field_type()->to_string(), names[i]); + } } TEST(ArrowSchemaUtilTest, IcebergParquetWriterPreservesUuidLogicalAnnotations) { diff --git a/be/test/format/table/iceberg/type_test.cpp b/be/test/format/table/iceberg/type_test.cpp index 81f8b6b760c24f..73bd2fd54df364 100644 --- a/be/test/format/table/iceberg/type_test.cpp +++ b/be/test/format/table/iceberg/type_test.cpp @@ -59,13 +59,34 @@ TEST(StringTypeTest, to_string) { } TEST(TimestampTypeTest, to_string) { - TimestampType timestamp_type_with_utc(true); + TimestampTzType timestamp_type_with_utc; EXPECT_EQ(timestamp_type_with_utc.to_string(), "timestamptz"); - EXPECT_EQ(timestamp_type_with_utc.type_id(), TypeID::TIMESTAMP); + EXPECT_EQ(timestamp_type_with_utc.type_id(), TypeID::TIMESTAMPTZ); - TimestampType timestamp_type_without_utc(false); + TimestampType timestamp_type_without_utc; EXPECT_EQ(timestamp_type_without_utc.to_string(), "timestamp"); EXPECT_EQ(timestamp_type_without_utc.type_id(), TypeID::TIMESTAMP); + + TimestampNsType timestamp_ns; + EXPECT_EQ(timestamp_ns.to_string(), "timestamp_ns"); + EXPECT_EQ(timestamp_ns.type_id(), TypeID::TIMESTAMP_NS); + + TimestampTzNsType timestamptz_ns; + EXPECT_EQ(timestamptz_ns.to_string(), "timestamptz_ns"); + EXPECT_EQ(timestamptz_ns.type_id(), TypeID::TIMESTAMPTZ_NS); +} + +TEST(TimestampTypeTest, ParsedTimestampKindsHaveDistinctIds) { + const char* names[] = {"timestamp", "timestamptz", "timestamp_ns", "timestamptz_ns"}; + // Type-based dispatch must preserve both the timestamp unit and timezone semantics. + for (int i = 0; i < 4; ++i) { + auto type = Types::from_primitive_string(names[i]); + EXPECT_EQ(type->to_string(), names[i]); + for (int j = 0; j < i; ++j) { + EXPECT_NE(type->type_id(), Types::from_primitive_string(names[j])->type_id()) + << names[i] << " and " << names[j]; + } + } } TEST(TimeTypeTest, to_string) { @@ -118,8 +139,10 @@ TEST(PrimitiveTypesTest, from_primitive_string) { EXPECT_EQ(Types::from_primitive_string("double")->type_id(), TypeID::DOUBLE); EXPECT_EQ(Types::from_primitive_string("date")->type_id(), TypeID::DATE); EXPECT_EQ(Types::from_primitive_string("time")->type_id(), TypeID::TIME); - EXPECT_EQ(Types::from_primitive_string("timestamptz")->type_id(), TypeID::TIMESTAMP); + EXPECT_EQ(Types::from_primitive_string("timestamptz")->type_id(), TypeID::TIMESTAMPTZ); EXPECT_EQ(Types::from_primitive_string("timestamp")->type_id(), TypeID::TIMESTAMP); + EXPECT_EQ(Types::from_primitive_string("timestamp_ns")->type_id(), TypeID::TIMESTAMP_NS); + EXPECT_EQ(Types::from_primitive_string("timestamptz_ns")->type_id(), TypeID::TIMESTAMPTZ_NS); EXPECT_EQ(Types::from_primitive_string("string")->type_id(), TypeID::STRING); EXPECT_EQ(Types::from_primitive_string("uuid")->type_id(), TypeID::UUID); EXPECT_EQ(Types::from_primitive_string("binary")->type_id(), TypeID::BINARY); From 98bda2479faa323fb60ec0b144b2b8c8c4174db3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 12 Oct 2026 15:29:06 +0800 Subject: [PATCH 04/13] [fix](datalake) Preserve nanoseconds across ORC reads and Iceberg operations ### What problem does this PR solve? Related PR: #68934 Problem Summary: External ORC writer timezones could shift fractional timestamps by one second when the nanosecond reader used its GMT carrier. Preserve ORC's correction at the writer epoch boundary. Keep Paimon physical metadata available when nanosecond scans use the SDK physical iterator. Use typed nanosecond literals for Iceberg pruning, filtered rewrites, static overwrite and conflict predicates, and preserve nanosecond defaults through ADD COLUMN and write planning. Accept date-only static partitions as midnight. ### Release note Fix nanosecond timestamp reads across ORC writer timezones, Paimon metadata queries, and Iceberg default, rewrite and static-partition operations. ### Check List (For Author) - Test: 270 focused FE tests, 263 ASAN BE tests, two local Iceberg/Paimon regression suites with generated goldens; FE Checkstyle, clang-format 16, clang-tidy and build hygiene. - Behavior changed: Yes. Preserve exact timestamp values and allow the affected nanosecond operations. - Does this need documentation: No. Correctness fixes for the nanosecond support in this PR. --- .../data_type_datetimev2_serde.cpp | 8 +- be/src/core/data_type_serde/data_type_serde.h | 2 + .../data_type_timestamp_ns_serde.cpp | 6 +- .../core/data_type_serde/orc_serde_utils.cpp | 13 +++ be/src/core/data_type_serde/orc_serde_utils.h | 3 + .../format_v2/orc/orc_file_input_stream.cpp | 1 + be/src/format_v2/orc/orc_file_input_stream.h | 2 + be/src/format_v2/orc/orc_reader.cpp | 20 +++++ be/test/format_v2/orc/orc_reader_test.cpp | 83 ++++++++++++++++++ .../iceberg/IcebergConnectorTransaction.java | 12 ++- .../iceberg/IcebergPartitionUtils.java | 19 ++-- .../iceberg/IcebergPredicateConverter.java | 21 +++++ .../iceberg/IcebergSchemaBuilder.java | 10 ++- .../iceberg/IcebergWriteSchemaContext.java | 2 + .../IcebergConnectorTransactionTest.java | 27 ++++++ .../iceberg/IcebergTimestampNsTest.java | 86 +++++++++++++++++++ .../rewrite/RewriteDataFilePlannerTest.java | 23 +++++ .../paimon/PaimonScanPlanProvider.java | 4 +- .../paimon/PaimonScanPlanProviderTest.java | 39 +++++++++ .../iceberg/test_iceberg_timestamp_ns.out | 14 +++ .../paimon/test_paimon_timestamp_ns.out | 24 ++++++ .../iceberg/test_iceberg_timestamp_ns.groovy | 24 +++++- .../paimon/test_paimon_timestamp_ns.groovy | 14 +++ 23 files changed, 440 insertions(+), 17 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 5edbeaceb290b4..6ea8c3a7c5c510 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -72,9 +72,11 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum } auto& value = reinterpret_cast&>(data[old_data_size + row]); + const int64_t seconds = orc_serde_utils::corrected_orc_timestamp_seconds( + orc_view, orc_batch->data[source_row], orc_batch->nanoseconds[source_row]); orc_serde_utils::RoundedOrcTimestamp timestamp; auto status = orc_serde_utils::round_orc_timestamp_to_microseconds( - orc_batch->data[source_row], orc_batch->nanoseconds[source_row], ×tamp); + seconds, orc_batch->nanoseconds[source_row], ×tamp); if (!status.ok()) { data.resize(old_data_size); return status; @@ -84,8 +86,8 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum // Instant carry precedes timezone conversion; plain TIMESTAMP carry stays in civil time // so rounding cannot jump across a daylight-saving gap or fold. status = orc_serde_utils::orc_timestamp_to_datetime( - is_timestamp_instant ? timestamp.seconds : orc_batch->data[source_row], - timestamp.microseconds, timezone, !is_timestamp_instant && timestamp.carry, &value); + is_timestamp_instant ? timestamp.seconds : seconds, timestamp.microseconds, + timezone, !is_timestamp_instant && timestamp.carry, &value); if (!status.ok()) { data.resize(old_data_size); return status; diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 51c215a524b48c..f45e16d378f969 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -126,6 +126,8 @@ struct OrcDecodedColumnView { const std::vector* selected_rows = nullptr; const cctz::time_zone* timezone = nullptr; bool enable_mapping_timestamp_tz = false; + // Writer's epoch-zero wall-clock offset, used only when ORC reads plain timestamps in GMT. + int32_t writer_epoch_offset = 0; }; // Deserialize means read from different file format or memory format, diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 96376a9e6911c0..65f9eaeb3d3ca2 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -490,8 +490,10 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, return Status::DataQualityError("Invalid ORC timestamp nanoseconds: {}", nanos); } DateV2Value civil; - auto status = orc_serde_utils::orc_timestamp_to_datetime(batch.data[source], nanos / 1000, - timezone, false, &civil); + const int64_t seconds = + orc_serde_utils::corrected_orc_timestamp_seconds(view, batch.data[source], nanos); + auto status = orc_serde_utils::orc_timestamp_to_datetime(seconds, nanos / 1000, timezone, + false, &civil); TimeStampNsValue value; if (!status.ok() || !value.from_datetime(civil, nanos % 1000)) { data.resize(old_size); diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp b/be/src/core/data_type_serde/orc_serde_utils.cpp index e4d78c2b7a2901..9dc13722bc2659 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.cpp +++ b/be/src/core/data_type_serde/orc_serde_utils.cpp @@ -36,6 +36,19 @@ bool has_nanosecond_timestamp(const ::orc::Type& type) { return false; } +int64_t corrected_orc_timestamp_seconds(const OrcDecodedColumnView& view, int64_t seconds, + int64_t nanoseconds) { + if (view.writer_epoch_offset == 0 || nanoseconds <= 999999 || + view.file_type->getKind() != ::orc::TIMESTAMP) { + return seconds; + } + // ORC 1.9 applies the negative-fraction correction after changing timezones. Undo it + // in the GMT carrier and apply it at the writer's epoch boundary instead, preserving + // fractions when the timezone shift crosses zero. TIMESTAMP_INSTANT is always UTC. + const int64_t encoded_seconds = seconds < 0 ? seconds + 1 : seconds; + return encoded_seconds < view.writer_epoch_offset ? encoded_seconds - 1 : encoded_seconds; +} + size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { if (selected_rows == nullptr) { return rows; diff --git a/be/src/core/data_type_serde/orc_serde_utils.h b/be/src/core/data_type_serde/orc_serde_utils.h index 00a2debe1bb4bb..14ec35531f1c6a 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.h +++ b/be/src/core/data_type_serde/orc_serde_utils.h @@ -32,6 +32,9 @@ namespace orc_serde_utils { bool has_nanosecond_timestamp(const ::orc::Type& type); +int64_t corrected_orc_timestamp_seconds(const OrcDecodedColumnView& view, int64_t seconds, + int64_t nanoseconds); + size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows); size_t orc_source_row_at(size_t row, const std::vector* selected_rows); bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row); diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index df0927aac3444a..57ef5a371d1508 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -278,6 +278,7 @@ void OrcFileInputStream::beforeReadStripe( std::unique_ptr<::orc::StripeInformation> current_stripe_information, const std::vector& selected_columns, std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams) { + _writer_timezone = current_stripe_information->getWriterTimezone(); _flush_active_clusters(); _active_stripe_streams.clear(); diff --git a/be/src/format_v2/orc/orc_file_input_stream.h b/be/src/format_v2/orc/orc_file_input_stream.h index 3ced728389004f..82ea64333ad5b2 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.h +++ b/be/src/format_v2/orc/orc_file_input_stream.h @@ -53,6 +53,7 @@ class OrcFileInputStream final : public ::orc::InputStream { uint64_t getNaturalReadSize() const override; void read(void* buf, uint64_t length, uint64_t offset) override; const std::string& getName() const override; + const std::string& writer_timezone() const { return _writer_timezone; } void beforeReadStripe(std::unique_ptr<::orc::StripeInformation> current_stripe_information, const std::vector& selected_columns, @@ -70,6 +71,7 @@ class OrcFileInputStream final : public ::orc::InputStream { std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams); std::string _file_name; + std::string _writer_timezone; io::FileReaderSPtr _file_reader; io::FileReaderSPtr _default_reader; const io::IOContext* _io_ctx = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index ad64e8dab7a9e7..6df78c655eca0b 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -763,6 +763,10 @@ struct OrcReaderScanState { ::orc::RowReaderOptions row_reader_options; // projection + filter + SARG + stripe range std::string timezone = TimezoneUtils::default_time_zone; cctz::time_zone timezone_obj; + bool utc_wall_clock = false; + OrcFileInputStream* input_stream = nullptr; + std::string stripe_writer_timezone; + int32_t writer_epoch_offset = 0; std::unique_ptr<::orc::RowReader> row_reader; const ::orc::Type* selected_type = nullptr; std::unique_ptr<::orc::ColumnVectorBatch> batch; @@ -951,11 +955,13 @@ Status OrcReader::init(RuntimeState* state) { auto input_stream = std::make_unique( _file_description->path, _file_reader, _io_ctx.get(), _profile, input_stream_options); try { + _state->input_stream = input_stream.get(); _state->reader = ::orc::createReader(std::move(input_stream), options); _state->root_type = &_state->reader->getType(); if (orc_serde_utils::has_nanosecond_timestamp(*_state->root_type)) { // Use the same UTC wall-clock carrier as the writer; historical local offsets // differ between ORC and cctz and can push boundary values out of range. + _state->utc_wall_clock = true; _state->timezone = "GMT"; _state->timezone_obj = cctz::utc_time_zone(); } @@ -1971,6 +1977,20 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type view.selected_rows = selected_rows; view.timezone = &_state->timezone_obj; view.enable_mapping_timestamp_tz = _enable_mapping_timestamp_tz; + if (_state->utc_wall_clock) { + const auto& writer_timezone = _state->input_stream->writer_timezone(); + if (_state->stripe_writer_timezone != writer_timezone) { + cctz::time_zone zone = cctz::utc_time_zone(); + if (!writer_timezone.empty() && + !TimezoneUtils::find_cctz_time_zone(writer_timezone, zone)) { + return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", + writer_timezone); + } + _state->writer_epoch_offset = zone.lookup(cctz::time_point {}).offset; + _state->stripe_writer_timezone = writer_timezone; + } + view.writer_epoch_offset = _state->writer_epoch_offset; + } return column_type->get_serde()->read_column_from_orc(*column, view); } diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 6ef065c9ff3951..e4c8a180606b20 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -10496,6 +10496,89 @@ TEST_F(NewOrcReaderTest, ReadTimestampNanosecondsRoundsToMicroseconds) { } } +TEST_F(NewOrcReaderTest, NanosecondFilesPreserveNegativeFractionsAcrossWriterTimezones) { + for (const auto& [writer_zone, offset] : std::vector> { + {"GMT", 0}, {"Asia/Shanghai", 28800}, {"America/New_York", -18000}}) { + SCOPED_TRACE(writer_zone); + const auto file_path = (_test_dir / "timestamp_ns_writer_zone.orc").string(); + auto type = ::orc::createStructType(); + auto ns_type = ::orc::createPrimitiveType(::orc::TIMESTAMP); + ns_type->setAttribute("iceberg.timestamp-unit", "NANOS"); + type->addStructField("ns", std::move(ns_type)); + type->addStructField("plain", ::orc::createPrimitiveType(::orc::TIMESTAMP)); + MemoryOutputStream stream(1024 * 1024); + ::orc::WriterOptions options; + options.setTimezoneName(writer_zone); + options.setCompression(::orc::CompressionKind_NONE); + auto writer = ::orc::createWriter(*type, &stream, options); + constexpr std::array fractions {0, 1, 999999, 1000000, 500000123, 999999999}; + constexpr size_t row_count = 3 * fractions.size(); + auto batch = writer->createRowBatch(row_count); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + root.numElements = row_count; + for (auto* field : root.fields) { + auto& timestamps = dynamic_cast<::orc::TimestampVectorBatch&>(*field); + timestamps.numElements = row_count; + for (size_t row = 0; row < row_count; ++row) { + // Use the external writer's epoch carrier, not Doris's UTC writer convention. + timestamps.data[row] = + (static_cast(row / fractions.size()) - 1) * 2 - offset; + timestamps.nanoseconds[row] = fractions[row % fractions.size()]; + } + } + writer->add(*batch); + writer->close(); + { + std::ofstream out(file_path, std::ios::binary); + out.write(stream.getData(), static_cast(stream.getLength())); + } + for (bool sparse : {false, true}) { + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("+08:00"); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + std::vector source_rows; + for (size_t row = 0; row < row_count; row += sparse ? 2 : 1) { + source_rows.push_back(static_cast(row)); + } + if (sparse) { + request->row_ids = source_rows; + } + ASSERT_TRUE(reader->open(request).ok()); + const std::array seconds {"1969-12-31 23:59:58", "1970-01-01 00:00:00", + "1970-01-01 00:00:02"}; + const std::array carried_seconds { + "1969-12-31 23:59:59", "1970-01-01 00:00:01", "1970-01-01 00:00:03"}; + const std::array micros {"000000", "000000", "001000", + "001000", "500000", "000000"}; + bool eof = false; + size_t output_row = 0; + while (!eof) { + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t row = 0; row < rows; ++row) { + ASSERT_LT(output_row, source_rows.size()); + const size_t source = static_cast(source_rows[output_row++]); + const size_t second_index = source / fractions.size(); + const size_t fraction_index = source % fractions.size(); + EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, row), + fmt::format("{}.{:09}", seconds[second_index], + fractions[fraction_index])); + const auto& whole = fraction_index == 5 ? carried_seconds : seconds; + EXPECT_EQ(schema[1].type->to_string(*block.get_by_position(1).column, row), + whole[second_index] + "." + micros[fraction_index]); + } + } + EXPECT_EQ(output_row, source_rows.size()); + } + } +} + Status decode_orc_timestamp_boundary(int64_t seconds, int64_t nanoseconds, bool file_is_instant, bool use_timestamp_tz, const cctz::time_zone& timezone = cctz::utc_time_zone(), diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index bd85c627e757b6..f1ea0923672406 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -66,6 +66,7 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ContentFileUtil; import org.apache.logging.log4j.LogManager; @@ -902,7 +903,7 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa } Object partitionValue = IcebergPartitionUtils.parsePartitionValueFromString( partitionValueStr, sourceField.type(), zone); - predicates.add(identityPartitionPredicate(sourceField.name(), partitionValue)); + predicates.add(identityPartitionPredicate(sourceField, partitionValue)); } } @@ -929,7 +930,8 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa * both paths that build this predicate: DELETE/UPDATE/MERGE conflict detection and * {@code INSERT OVERWRITE ... PARTITION(d='nan')}. */ - private static Expression identityPartitionPredicate(String sourceColName, Object value) { + private static Expression identityPartitionPredicate(Types.NestedField sourceField, Object value) { + String sourceColName = sourceField.name(); if (value == null) { return Expressions.isNull(sourceColName); } @@ -937,6 +939,10 @@ private static Expression identityPartitionPredicate(String sourceColName, Objec && Double.isNaN(((Number) value).doubleValue())) { return Expressions.isNaN(sourceColName); } + if (sourceField.type().typeId() == TypeID.TIMESTAMP_NANO) { + // Partition values are already nanos; a raw Long would be rebound as microseconds. + return Expressions.equal(sourceColName, Expressions.nanos((Long) value)); + } return Expressions.equal(sourceColName, value); } @@ -1223,7 +1229,7 @@ private Expression buildIdentityPartitionExpression(PartitionSpec spec, Schema s valueStr = null; } Object value = IcebergPartitionUtils.parsePartitionValueFromString(valueStr, sourceField.type(), zone); - Expression predicate = identityPartitionPredicate(sourceField.name(), value); + Expression predicate = identityPartitionPredicate(sourceField, value); expression = expression == null ? predicate : Expressions.and(expression, predicate); } return expression; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index edce2f0463e3c0..95a5aa4c935ec5 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -492,12 +492,11 @@ static Object parsePartitionValueFromString(String valueStr, Type icebergType, Z // Iceberg date is days since epoch (1970-01-01). return (int) LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay(); case TIMESTAMP_NANO: - LocalDateTime timestampNs = LocalDateTime.parse(valueStr.replace('T', ' '), - TIMESTAMP_PARTITION_FORMAT); - // Widen before combining: the minimum nanosecond timestamp's whole second overflows long. - return BigDecimal.valueOf(timestampNs.toEpochSecond(ZoneOffset.UTC)) - .multiply(BigDecimal.valueOf(1_000_000_000L)) - .add(BigDecimal.valueOf(timestampNs.getNano())).longValueExact(); + // Match the sink cast: a date-only static partition denotes midnight. + LocalDateTime timestampNs = valueStr.length() == 10 + ? LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).atStartOfDay() + : LocalDateTime.parse(valueStr.replace('T', ' '), TIMESTAMP_PARTITION_FORMAT); + return toEpochNanos(timestampNs); case TIMESTAMP: return parseTimestampToMicros(valueStr, (TimestampType) icebergType, zone); case DECIMAL: @@ -1161,4 +1160,12 @@ List getUpperBound() { return upperBound; } } + + static long toEpochNanos(LocalDateTime timestamp) { + // Widen before combining: the minimum nanosecond timestamp's whole second overflows long. + return BigDecimal.valueOf(timestamp.toEpochSecond(ZoneOffset.UTC)) + .multiply(BigDecimal.valueOf(1_000_000_000L)) + .add(BigDecimal.valueOf(timestamp.getNano())).longValueExact(); + } + } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java index e5b77877e605b0..b0a96fae59864e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java @@ -32,6 +32,7 @@ import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.expressions.Not; import org.apache.iceberg.expressions.Or; import org.apache.iceberg.expressions.Unbound; @@ -509,6 +510,8 @@ private Object extractIcebergLiteral(Type icebergType, ConnectorLiteral literal) case STRING: case DATE: return date.toString(); + case TIMESTAMP_NANO: + return toNanos(date.atStartOfDay(), icebergType); case TIMESTAMP: return toMicros(date.atStartOfDay(), icebergType); default: @@ -520,6 +523,8 @@ private Object extractIcebergLiteral(Type icebergType, ConnectorLiteral literal) case STRING: case DATE: return dorisDateTimeString(dateTime); + case TIMESTAMP_NANO: + return toNanos(dateTime, icebergType); case TIMESTAMP: return toMicros(dateTime, icebergType, "TIMESTAMPTZ".equalsIgnoreCase(literal.getType().getTypeName())); @@ -603,6 +608,9 @@ private Object extractIcebergLiteral(Type icebergType, ConnectorLiteral literal) } else if (value instanceof String) { String string = (String) value; switch (id) { + case TIMESTAMP_NANO: + LocalDateTime parsedNs = parseDorisDateTime(string); + return parsedNs == null ? null : toNanos(parsedNs, icebergType); case TIMESTAMP: if (mode == Mode.SCAN) { // Legacy IcebergUtils.extractDorisLiteral returns the raw string for a TIMESTAMP column and @@ -671,6 +679,19 @@ private long toMicros(LocalDateTime dateTime, Type icebergType, boolean utcCarri return instant.getEpochSecond() * 1_000_000L + instant.getNano() / 1000L; } + private static Literal toNanos(LocalDateTime dateTime, Type icebergType) { + if (((Types.TimestampNanoType) icebergType).shouldAdjustToUTC()) { + return null; + } + try { + // A raw Long is a microsecond literal in Iceberg; preserve the nanosecond unit explicitly. + // Unzoned values use a UTC wall-clock carrier, never the session zone. + return Expressions.nanos(IcebergPartitionUtils.toEpochNanos(dateTime)); + } catch (ArithmeticException e) { + return null; + } + } + // Parse a Doris-format datetime string ("yyyy-MM-dd HH:mm:ss[.fraction]") -- or a date-only string, taken // at start-of-day -- to a LocalDateTime, or null when unparseable. Normalizes the space separator to ISO // 'T' so java.time's ISO parsers accept the Doris rendering (mirrors legacy IcebergNereidsUtils' string -> diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java index 973528d0f7e482..c5860163cf363a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaBuilder.java @@ -31,12 +31,14 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -360,10 +362,16 @@ public static Literal parseDefaultLiteral(String value, Type type) { } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid Int string: " + value, e); } + case TIMESTAMP_NANO: + // ADD COLUMN carries the validated SQL datetime, not an epoch integer. + if (!value.matches("[+-]?[0-9]+")) { + return Expressions.nanos((Long) IcebergPartitionUtils.parsePartitionValueFromString( + value, type, ZoneOffset.UTC)); + } + return Expressions.nanos(Long.parseLong(value)); case LONG: case TIME: case TIMESTAMP: - case TIMESTAMP_NANO: try { return Literal.of(Long.parseLong(value)); } catch (NumberFormatException e) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java index 92bd5904db461b..12c592c8fa1ec8 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java @@ -439,6 +439,8 @@ static String toDorisSql(Type type, Object value, return binarySql(byteBufferBytes((ByteBuffer) value), enableMappingVarbinary); case DATE: return quote(LocalDate.ofEpochDay(((Integer) value).longValue()).toString()); + case TIMESTAMP_NANO: + // Identity formatting preserves all nine digits, including negative epoch fractions. case TIMESTAMP: String timestamp = Transforms.identity(type).toHumanString(type, value).replace('T', ' '); // Zoned defaults retain their offset regardless of the legacy catalog flag. diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 35a4ae5ad29195..739eb1aecb5365 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -43,6 +43,7 @@ import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -888,6 +889,32 @@ public void overwriteStaticPartitionUsesRowFilter() { Assertions.assertEquals("1", snap.summary().get("added-data-files")); } + @Test + public void dateOnlyNanosecondStaticOverwriteKeepsAdjacentPartition() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("ts").build(); + Table table = catalog.createTable(id, schema, spec, props("format-version", "3")); + long midnight = 1704067200000000000L; + for (long nanos : new long[] {midnight, midnight + 1}) { + PartitionData partition = new PartitionData(spec.partitionType()); + partition.set(0, nanos); + table.newAppend().appendFile(DataFiles.builder(spec) + .withPath("s3://bucket/" + nanos + ".parquet").withFileSizeInBytes(1024) + .withRecordCount(1).withPartition(partition).withFormat(FileFormat.PARQUET).build()).commit(); + } + IcebergConnectorTransaction txn = txnFor(opsReturning(table), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteStaticCtx(table, Collections.singletonMap("ts", "2024-01-01"))); + txn.addCommitData(commitBytes(dataFileItem("s3://bucket/replacement.parquet", 1L, 1024L, + Collections.singletonList("2024-01-01 00:00:00.000000000")))); + txn.commit(); + List remaining = currentDataFiles(catalog.loadTable(id)); + Assertions.assertEquals(2, remaining.size()); + Assertions.assertTrue(remaining.stream().anyMatch(file -> file.partition().get(0, Long.class) == midnight + 1)); + Assertions.assertTrue(remaining.stream().anyMatch(file -> file.path().toString().endsWith("replacement.parquet"))); + } + @Test public void overwriteStaticPartitionRejectsUnmatchedPartitionField() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java index 2d24fc96e16ce9..3d87eebf7a0853 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java @@ -18,19 +18,27 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.spi.pushdown.ConnectorComparison; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.inmemory.InMemoryCatalog; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Collections; +import java.util.List; +import java.util.Optional; public class IcebergTimestampNsTest { @Test @@ -62,6 +70,84 @@ public void nanosecondTimestampRoundTripsWithoutTimezone() { Types.TimestampNanoType.withZone(), false, false).getTypeName()); } + @Test + public void predicatesPreserveNanosecondsInEveryMode() { + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); + for (IcebergPredicateConverter.Mode mode : IcebergPredicateConverter.Mode.values()) { + for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) { + IcebergPredicateConverter converter = new IcebergPredicateConverter(schema, ZoneId.of(zone), mode); + for (String text : new String[] {"2024-01-02 03:04:05.123456789", + "1969-12-31 23:59:59.999999999", "1677-09-21 00:12:43.145224192", + "2262-04-11 23:47:16.854775807"}) { + long expected = (long) IcebergPartitionUtils.parsePartitionValueFromString( + text, Types.TimestampNanoType.withoutZone(), ZoneId.of(zone)); + for (ConnectorLiteral literal : new ConnectorLiteral[] { + new ConnectorLiteral(ConnectorType.of("STRING"), text), + new ConnectorLiteral(ConnectorType.of("TIMESTAMP_NS"), + LocalDateTime.parse(text.replace(' ', 'T')))}) { + List expressions = converter.convert(new ConnectorComparison( + ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), literal)); + Assertions.assertEquals(1, expressions.size(), mode + " " + text); + Assertions.assertEquals(expected, + ((UnboundPredicate) expressions.get(0)).literal().value()); + } + } + } + } + } + + @Test + public void writeDefaultsPreserveNanoseconds() { + for (long nanos : new long[] {Long.MIN_VALUE, -1, 0, 1, 123456789, Long.MAX_VALUE}) { + String sql = IcebergWriteSchemaContext.toDorisSql( + Types.TimestampNanoType.withoutZone(), nanos, false, false); + Assertions.assertTrue(sql.startsWith("'") && sql.endsWith("'")); + String text = sql.substring(1, sql.length() - 1); + Assertions.assertEquals(nanos, IcebergPartitionUtils.parsePartitionValueFromString( + text, Types.TimestampNanoType.withoutZone(), ZoneId.of("UTC"))); + } + } + + @Test + public void addColumnDefaultsAcceptNanosecondDateTimes() { + String text = "2024-01-02 03:04:05.123456789"; + Assertions.assertEquals(1704164645123456789L, IcebergSchemaBuilder.parseDefaultLiteral( + text, Types.TimestampNanoType.withoutZone()).value()); + Assertions.assertEquals(Long.MIN_VALUE, IcebergSchemaBuilder.parseDefaultLiteral( + "1677-09-21 00:12:43.145224192", Types.TimestampNanoType.withoutZone()).value()); + Assertions.assertThrows(IllegalArgumentException.class, () -> IcebergSchemaBuilder.parseDefaultLiteral( + "2262-04-11 23:47:16.854775808", Types.TimestampNanoType.withoutZone())); + } + + @Test + public void addColumnDefaultSurvivesSchemaCommitAndWritePlanning() { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db")); + Table table = catalog.createTable(TableIdentifier.of("db", "defaults"), + new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())), + PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "3")); + String text = "2024-01-02 03:04:05.123456789"; + table.updateSchema().addColumn("ts", Types.TimestampNanoType.withoutZone(), null, + IcebergSchemaBuilder.parseDefaultLiteral(text, Types.TimestampNanoType.withoutZone())).commit(); + Assertions.assertEquals(1704164645123456789L, table.schema().findField("ts").initialDefault()); + Assertions.assertEquals(1704164645123456789L, table.schema().findField("ts").writeDefault()); + IcebergWriteSchemaContext context = IcebergWriteSchemaContext.create( + table, "defaults", Optional.empty(), false, false); + Assertions.assertEquals("'" + text + "'", context.getColumns().get(1).getDefaultValueSql()); + } + + @Test + public void staticIdentityPartitionAcceptsDateOnly() { + for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) { + Object expected = IcebergPartitionUtils.parsePartitionValueFromString( + "2024-01-01 00:00:00", Types.TimestampNanoType.withoutZone(), ZoneId.of(zone)); + Assertions.assertEquals(expected, IcebergPartitionUtils.parsePartitionValueFromString( + "2024-01-01", Types.TimestampNanoType.withoutZone(), ZoneId.of(zone))); + } + } + @Test public void identityPartitionsPreserveNanosecondsAcrossEpochAndTimezones() { for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlannerTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlannerTest.java index 5e941a30de1365..e2b8005079f104 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlannerTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlannerTest.java @@ -35,6 +35,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -427,6 +428,28 @@ public void partiallyPushableWhereThrows() { Assertions.assertTrue(e.getMessage().contains("cannot be pushed down")); } + @Test + public void nanosecondWhereRewritesOnlyTheMatchingPartition() { + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("ts").build(); + Table table = catalog.createTable(TableIdentifier.of("db1", "ns"), schema, spec, + Collections.singletonMap(TableProperties.FORMAT_VERSION, "3")); + long expected = 1704164645123456789L; + for (long nanos : new long[] {expected, expected + 1}) { + PartitionData partition = new PartitionData(spec.partitionType()); + partition.set(0, nanos); + append(table, DataFiles.builder(spec).withPath("s3://bucket/" + nanos + ".parquet") + .withFileSizeInBytes(100).withRecordCount(1).withPartition(partition) + .withFormat(FileFormat.PARQUET).build()); + } + ConnectorExpression predicate = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), + new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-02 03:04:05.123456789")); + List groups = plan(table, rewriteAll(1_000_000L, where(predicate))); + Assertions.assertEquals(1, totalFiles(groups)); + Assertions.assertEquals(expected, groups.get(0).getTasks().get(0).file().partition().get(0, Long.class)); + } + @Test public void whereBetweenPrunesToMatchingPartition() { // BETWEEN is a node the rewrite-side neutral converter emits directly; REWRITE mode pushes it diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 4d494696f46358..3790c7f9daaba0 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -1019,9 +1019,11 @@ && hasTrustworthyLimitAccounting(table) // FIX-L14: ignore_split_type=IGNORE_JNI drops JNI splits (legacy getSplits:483). continue; } + // Lossless timestamp fallbacks can expose metadata through the SDK's physical iterator. // Raw convertibility alone does not select the SDK raw reader: historical primary-key // files without delete counts require merging and lose physical file/row positions. - if (requiresMetadataColumns && (!legacyOrcTimestamp || !supportsJniPhysicalMetadata( + if (requiresMetadataColumns && (!(legacyOrcTimestamp || needsNanosecondReader) + || !supportsJniPhysicalMetadata( legacyOrcSchemaTable.get(), dataSplit))) { validateMetadataColumnReader(true, false); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 0ce6227673a3c4..36d793c5acd62d 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -2821,6 +2821,45 @@ public void determineTargetSplitSizeMirrorsLegacyHeuristic() { "max_file_split_num floor (ceil(total/maxNum)) must raise the target above 64MB"); } + @Test + public void nanosecondColumnsRetainPhysicalMetadata(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + for (String format : Arrays.asList("parquet", "orc")) { + Identifier id = Identifier.create("db", "events_" + format); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", new org.apache.paimon.types.TimestampType(9)) + .option("file.format", format).build(), false); + Table table = catalog.getTable(id); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite()) { + write.write(GenericRow.of(1, org.apache.paimon.data.Timestamp.fromLocalDateTime( + java.time.LocalDateTime.parse("2024-01-01T00:00:00.123456789")))); + try (BatchTableCommit commit = builder.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle( + "db", id.getObjectName(), Collections.emptyList(), Collections.emptyList()); + List ranges = provider.planScan(sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handle, Arrays.asList( + new PaimonColumnHandle("event_time", 1), + new PaimonColumnHandle(PAIMON_FILE_PATH_COL, -1), + new PaimonColumnHandle(PAIMON_ROW_POSITION_COL, -1))).build()); + Assertions.assertFalse(ranges.isEmpty()); + for (ConnectorScanRange range : ranges) { + Assertions.assertTrue(range.getProperties().containsKey("paimon.split")); + } + } + } + } + @Test public void legacyPrimaryKeyFilesCannotExposePhysicalMetadata(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out index 358ac94738fc0c..f3a38239c85848 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -57,6 +57,10 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N +-- !parquet_defaults -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 + -- !parquet_identity -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 @@ -75,6 +79,9 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 7 2262-04-11 23:47:16.854775807 +-- !parquet_identity_static_date -- +2024-01-01 00:00:00.000000000 + -- !parquet_year -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 @@ -218,6 +225,10 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 \N \N \N 7 2262-04-11 23:47:16.854775807 \N \N \N +-- !orc_defaults -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 + -- !orc_identity -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 @@ -236,6 +247,9 @@ entries map Yes true \N 6 1677-09-21 00:12:43.145224192 7 2262-04-11 23:47:16.854775807 +-- !orc_identity_static_date -- +2024-01-01 00:00:00.000000000 + -- !orc_year -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out index 7d4921ac9809e0..33170ff4b5e8cd 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_ns.out @@ -150,6 +150,18 @@ entries map Yes true \N -- !ns_parquet_false_true_true_Asia_Shanghai_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 +-- !ns_parquet_false_metadata -- +1 2024-01-02 03:04:05.123456789 true true +2 2024-01-02 03:04:05.123456788 true true +3 1969-12-31 23:59:58.999999999 true true +4 1970-01-01 00:00:00.000000001 true true +5 \N true true +6 1677-09-21 00:12:43.145224192 true true +7 2262-04-11 23:47:16.854775807 true true + +-- !ns_parquet_false_metadata_unique -- +7 + -- !ns_parquet_false_copy -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} @@ -479,6 +491,18 @@ entries map Yes true \N -- !ns_orc_false_true_true_Asia_Shanghai_aggregate -- 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 6 +-- !ns_orc_false_metadata -- +1 2024-01-02 03:04:05.123456789 true true +2 2024-01-02 03:04:05.123456788 true true +3 1969-12-31 23:59:58.999999999 true true +4 1970-01-01 00:00:00.000000001 true true +5 \N true true +6 1677-09-21 00:12:43.145224192 true true +7 2262-04-11 23:47:16.854775807 true true + +-- !ns_orc_false_metadata_unique -- +7 + -- !ns_orc_false_copy -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy index a89616554ffba0..4e003f850133a5 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -20,7 +20,8 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { return } String host = context.config.otherConfigs.get("externalEnvIp") - String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + // The newer REST fixture preserves nanosecond schema defaults; the baseline 1.10 server rescales them. + String restPort = context.config.otherConfigs.get("iceberg_scan_planning_rest_uri_port") String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String catalog = "test_iceberg_timestamp_ns" def zone = sql("SELECT @@time_zone")[0][0] @@ -68,6 +69,22 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { sql """CREATE TABLE ${table}_copy PROPERTIES ("format-version"="3", "write.format.default"="${format}") AS SELECT * FROM ${table}""" "qt_${format}_copy" "SELECT * FROM ${table}_copy ORDER BY id" + // Filtered rewrites must bind a nanosecond literal without scaling it as microseconds. + def beforeRewrite = sql("SELECT * FROM ${table} ORDER BY id") + sql """ALTER TABLE ${table} EXECUTE rewrite_data_files("rewrite-all"="true") + WHERE ts='2024-01-02 03:04:05.123456789'""" + assertEquals(beforeRewrite, sql("SELECT * FROM ${table} ORDER BY id")) + + String defaultsTable = "${table}_defaults" + sql "DROP TABLE IF EXISTS ${defaultsTable}" + sql """CREATE TABLE ${defaultsTable} (id INT) PROPERTIES + ("format-version"="3", "write.format.default"="${format}")""" + // Exercise both metadata default parsing and the subsequent statement-scoped sink schema. + sql """ALTER TABLE ${defaultsTable} ADD COLUMN ts TIMESTAMP_NS + DEFAULT '2024-01-02 03:04:05.123456789'""" + sql "INSERT INTO ${defaultsTable} (id) VALUES (1)" + sql "INSERT INTO ${defaultsTable} VALUES (2, '2024-01-02 03:04:05.123456788')" + "qt_${format}_defaults" "SELECT id, CAST(ts AS STRING) FROM ${defaultsTable} ORDER BY id" for (String transform : ["identity", "year", "month", "day", "hour", "bucket"]) { String partitionTable = "${table}_${transform}" String expression = transform == "identity" ? "ts" @@ -80,6 +97,11 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { "qt_${format}_${transform}" "SELECT * FROM ${partitionTable} ORDER BY id" sql "INSERT OVERWRITE TABLE ${partitionTable} SELECT id, ts FROM ${table} WHERE id=2" "qt_${format}_${transform}_overwrite" "SELECT * FROM ${partitionTable} ORDER BY id" + if (transform == "identity") { + // Static partition commit parsing must accept the same date-only input as the sink cast. + sql "INSERT OVERWRITE TABLE ${partitionTable} PARTITION(ts='2024-01-01') SELECT 8" + "qt_${format}_identity_static_date" "SELECT CAST(ts AS STRING) FROM ${partitionTable} WHERE id=8" + } } } } finally { diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy index 1c9acb0bdd685c..2fd93642490f35 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_ns.groovy @@ -71,6 +71,20 @@ suite("test_paimon_timestamp_ns", "p0,external,paimon") { } } } + if (!primaryKey) { + sql "SET enable_file_scanner_v2=true" + sql "SET force_jni_scanner=false" + // Automatic lossless JNI reads must retain physical metadata, including after filtering. + def metadata = sql("""SELECT id, CAST(ts AS STRING), __paimon_file_path, __paimon_row_index + FROM ${table} ORDER BY id""") + "qt_${table}_metadata" """SELECT id, CAST(ts AS STRING), + __paimon_file_path IS NOT NULL, __paimon_row_index >= 0 FROM ${table} ORDER BY id""" + "qt_${table}_metadata_unique" """SELECT COUNT(*) FROM + (SELECT DISTINCT __paimon_file_path, __paimon_row_index FROM ${table}) positions""" + assertEquals([metadata[0]], sql("""SELECT id, CAST(ts AS STRING), + __paimon_file_path, __paimon_row_index FROM ${table} + WHERE ts=CAST('2024-01-02 03:04:05.123456789' AS TIMESTAMP_NS)""")) + } sql "DROP TABLE IF EXISTS ${table}_copy" sql """CREATE TABLE ${table}_copy PROPERTIES ("file.format"="${format}") AS SELECT * FROM ${table}""" From 37c6355deac92b9bded1e263f2a8ccc5e6b665ff Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 12 Oct 2026 16:06:27 +0800 Subject: [PATCH 05/13] [fix](regression) Align Paimon timestamp expectations with nanosecond mapping ### What problem does this PR solve? Related PR: #68934 Problem Summary: Paimon precision-7/8/9 unzoned timestamps now map to TIMESTAMP_NS, but the MinIO, SHOW CREATE TABLE and scanner-V2-disabled regression expectations still described microsecond values. Retain the upstream scanner coverage and regenerate these results by running the three suites against the external Paimon fixtures. Clarify that automatic JNI selection preserves nanoseconds regardless of the scanner V2 setting. ### Release note None ### Check List (For Author) - Test: Run paimon_timestamp_types, test_paimon_minio and test_paimon_table_properties locally; generate their outputs through the regression runner and rerun for comparison. - Behavior changed: No. Align regression expectations with the nanosecond mapping in this PR. - Does this need documentation: No. --- .../external_table_p0/paimon/paimon_timestamp_types.out | 4 ++-- .../data/external_table_p0/paimon/test_paimon_minio.out | 9 +++++---- .../paimon/test_paimon_table_properties.out | 3 ++- .../paimon/paimon_timestamp_types.groovy | 5 ++--- .../external_table_p0/paimon/test_paimon_minio.groovy | 2 +- .../paimon/test_paimon_table_properties.groovy | 2 +- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out index c86454f6243eaf..ec7e602af4b950 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out +++ b/regression-test/data/external_table_p0/paimon/paimon_timestamp_types.out @@ -96,10 +96,10 @@ 1 -- !c1_v1 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !c2_v1 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !ltz_ntz_simple2 -- 1 {"2024-01-01 10:12:34.123456":"2024-01-02 10:12:34.123456", "2024-01-03 10:12:34.123456":"2024-01-04 10:12:34.123456"} {"2024-01-03 10:12:34.123456+08:00":"2024-01-04 10:12:34.123456+08:00", "2024-01-01 10:12:34.123456+08:00":"2024-01-02 10:12:34.123456+08:00"} ["2024-01-01 10:12:34.123456", "2024-01-02 10:12:34.123456", "2024-01-03 10:12:34.123456"] ["2024-01-01 10:12:34.123456+08:00", "2024-01-02 10:12:34.123456+08:00", "2024-01-03 10:12:34.123456+08:00"] {"crow1":"2024-01-01 10:12:34.123456", "crow2":"2024-01-02 10:12:34.123456+08:00"} diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out index 02bdb4e7eaa2a1..75156017a650ce 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out @@ -3,22 +3,23 @@ 2024-01-02T10:04:05.100 2024-01-02 10:04:05.123456+08:00 -- !no_region2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !region1 -- 2024-01-02T10:04:05.100 2024-01-02 10:04:05.123456+08:00 -- !region2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !no_region1 -- 2024-01-02T10:04:05.100 2024-01-02 10:04:05.123456+08:00 -- !no_region2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 -- !region1 -- 2024-01-02T10:04:05.100 2024-01-02 10:04:05.123456+08:00 -- !region2 -- -1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 +1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02 10:04:05.123456700 2024-01-02 10:04:05.123456780 2024-01-02 10:04:05.123456789 2024-01-02 10:04:05.1+08:00 2024-01-02 10:04:05.12+08:00 2024-01-02 10:04:05.123+08:00 2024-01-02 10:04:05.1234+08:00 2024-01-02 10:04:05.12345+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 2024-01-02 10:04:05.123456+08:00 + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_table_properties.out b/regression-test/data/external_table_p0/paimon/test_paimon_table_properties.out index 7a52529a1d0917..e4bbd432489974 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_table_properties.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_table_properties.out @@ -1,3 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show_create_table -- -ts_scale_orc CREATE TABLE `ts_scale_orc` (\n `id` int NULL,\n `ts1` datetimev2(1) NULL,\n `ts2` datetimev2(2) NULL,\n `ts3` datetimev2(3) NULL,\n `ts4` datetimev2(4) NULL,\n `ts5` datetimev2(5) NULL,\n `ts6` datetimev2(6) NULL,\n `ts7` datetimev2(6) NULL,\n `ts8` datetimev2(6) NULL,\n `ts9` datetimev2(6) NULL,\n `ts11` timestamptz(1) NULL,\n `ts12` timestamptz(2) NULL,\n `ts13` timestamptz(3) NULL,\n `ts14` timestamptz(4) NULL,\n `ts15` timestamptz(5) NULL,\n `ts16` timestamptz(6) NULL,\n `ts17` timestamptz(6) NULL,\n `ts18` timestamptz(6) NULL,\n `ts19` timestamptz(6) NULL\n) ENGINE=Paimon\nLOCATION 's3://warehouse/wh/flink_paimon.db/ts_scale_orc'\nPROPERTIES (\n "path" = "s3://warehouse/wh/flink_paimon.db/ts_scale_orc",\n "write-only" = "true",\n "file.format" = "orc"\n); +ts_scale_orc CREATE TABLE `ts_scale_orc` (\n `id` int NULL,\n `ts1` datetimev2(1) NULL,\n `ts2` datetimev2(2) NULL,\n `ts3` datetimev2(3) NULL,\n `ts4` datetimev2(4) NULL,\n `ts5` datetimev2(5) NULL,\n `ts6` datetimev2(6) NULL,\n `ts7` timestamp_ns NULL,\n `ts8` timestamp_ns NULL,\n `ts9` timestamp_ns NULL,\n `ts11` timestamptz(1) NULL,\n `ts12` timestamptz(2) NULL,\n `ts13` timestamptz(3) NULL,\n `ts14` timestamptz(4) NULL,\n `ts15` timestamptz(5) NULL,\n `ts16` timestamptz(6) NULL,\n `ts17` timestamptz(6) NULL,\n `ts18` timestamptz(6) NULL,\n `ts19` timestamptz(6) NULL\n) ENGINE=Paimon\nLOCATION 's3://warehouse/wh/flink_paimon.db/ts_scale_orc'\nPROPERTIES (\n "path" = "s3://warehouse/wh/flink_paimon.db/ts_scale_orc",\n "write-only" = "true",\n "file.format" = "orc"\n); + diff --git a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy index b78ebbdbac6fdc..b3e3f35873a85d 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_timestamp_types.groovy @@ -158,9 +158,8 @@ suite("paimon_timestamp_types", "p0,external") { test_scale() // Keep testing legacy LTZ fallback with V2 disabled independently of the NTZ predicate tests. sql """set enable_file_scanner_v2=false""" - // V1 must still decode every timestamp scale. Legacy ORC LTZ files need Paimon's - // compatibility conversion even when JNI is not forced, while Parquet stays on the - // native V1 reader. The ts9 predicates above stay on V2 only. + // High-precision NTZ columns require JNI even with V2 disabled; the fallback must + // preserve all nine digits instead of retaining the old microsecond expectations. order_qt_c1_v1 """select * from ts_scale_orc""" order_qt_c2_v1 """select * from ts_scale_parquet""" // test_ltz_ntz("test_timestamp_ntz_ltz_orc") diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy index 07e8ce9382fc51..c23bdadd9d7fe7 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy @@ -40,7 +40,7 @@ suite("test_paimon_minio", "p0,external") { sql """switch `${catalog_name}`""" sql """show databases; """ sql """use `${catalog_name}`.`flink_paimon`""" - // Native ORC rounds the fixture's nanosecond fractions to six decimal places. + // NTZ scales 7-9 retain nanoseconds through JNI; LTZ values keep microsecond precision. order_qt_no_region1 """select ts1,ts19 from ${table_name} """ order_qt_no_region2 """select * from ${table_name} """ // 3.1 new features diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_table_properties.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_table_properties.groovy index 9b7bc89b4337e7..5562052afbc061 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_table_properties.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_table_properties.groovy @@ -37,7 +37,7 @@ suite("test_paimon_table_properties", "p0,external") { """ sql """switch `${catalog_name}`""" sql """use `${catalog_name}`.`flink_paimon`""" - // SHOW CREATE must retain the LTZ/NTZ distinction from the Paimon logical schema. + // SHOW CREATE must retain NTZ scales 7-9 as TIMESTAMP_NS, distinct from microsecond LTZ. qt_show_create_table """show create table ${table_name} """ sql """drop catalog if exists ${catalog_name}_with_region""" From bdd0c0a1a2bed81d5d74aee48545b840b2fb21ef Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 12 Oct 2026 18:43:58 +0800 Subject: [PATCH 06/13] [fix](external) Correct nanosecond defaults, precision and ORC decoding ### What problem does this PR solve? Related PR: #68934 Problem Summary: Normalize compact civil TIMESTAMP_NS SQL defaults before connector persistence. Truncate Paimon nanosecond transport values to the target precision before key hashing and serialization. Preserve ORC writer timestamps through historical timezone transitions and negative fractional seconds, while avoiding sparse timestamp decoding in affected files and retaining the session timezone for instant columns. ### Release note Fix compact external timestamp defaults, Paimon TIMESTAMP(7/8) writes, and historical or filtered ORC timestamp reads in nanosecond-annotated files. ### Check List (For Author) - Test: FE default and Paimon adapter/schema unit tests; 211 related BE tests; Iceberg TIMESTAMP_NS regression with generated and verified output; FE Checkstyle, clang-format 16, changed-line clang-tidy and build hygiene. - Behavior changed: Yes, preserve civil defaults, declared precision and historical timestamp values. - Does this need documentation: No --- .../data_type_datetimev2_serde.cpp | 9 +- be/src/core/data_type_serde/data_type_serde.h | 4 +- .../data_type_timestamp_ns_serde.cpp | 8 +- .../core/data_type_serde/orc_serde_utils.cpp | 13 -- be/src/core/data_type_serde/orc_serde_utils.h | 3 - .../format_v2/orc/orc_file_input_stream.cpp | 1 - be/src/format_v2/orc/orc_file_input_stream.h | 2 - be/src/format_v2/orc/orc_reader.cpp | 197 ++++++++++++++---- be/src/format_v2/orc/orc_reader.h | 3 +- be/test/format_v2/orc/orc_reader_test.cpp | 132 ++++++++++++ .../doris/paimon/PaimonArrowBatchAdapter.java | 16 +- .../paimon/PaimonArrowBatchAdapterTest.java | 96 +++++++++ .../plans/commands/info/ColumnDefinition.java | 8 + .../commands/info/ColumnDefinitionTest.java | 17 ++ .../iceberg/test_iceberg_timestamp_ns.out | 10 + .../iceberg/test_iceberg_timestamp_ns.groovy | 7 + 16 files changed, 448 insertions(+), 78 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 6ea8c3a7c5c510..acac0d6e385fe6 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -72,8 +72,7 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum } auto& value = reinterpret_cast&>(data[old_data_size + row]); - const int64_t seconds = orc_serde_utils::corrected_orc_timestamp_seconds( - orc_view, orc_batch->data[source_row], orc_batch->nanoseconds[source_row]); + const int64_t seconds = orc_batch->data[source_row]; orc_serde_utils::RoundedOrcTimestamp timestamp; auto status = orc_serde_utils::round_orc_timestamp_to_microseconds( seconds, orc_batch->nanoseconds[source_row], ×tamp); @@ -1051,7 +1050,11 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { return Status::OK(); } - return decode_timestamp_orc_values(column, view, *view.timezone); + const auto* timezone = view.file_type->getKind() == ::orc::TIMESTAMP && + view.plain_timestamp_timezone != nullptr + ? view.plain_timestamp_timezone + : view.timezone; + return decode_timestamp_orc_values(column, view, *timezone); } } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index f45e16d378f969..b7df1b019bc5c3 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -126,8 +126,8 @@ struct OrcDecodedColumnView { const std::vector* selected_rows = nullptr; const cctz::time_zone* timezone = nullptr; bool enable_mapping_timestamp_tz = false; - // Writer's epoch-zero wall-clock offset, used only when ORC reads plain timestamps in GMT. - int32_t writer_epoch_offset = 0; + // Plain timestamps may retain the writer's carrier; instant timestamps still use the session zone. + const cctz::time_zone* plain_timestamp_timezone = nullptr; }; // Deserialize means read from different file format or memory format, diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 65f9eaeb3d3ca2..7fd8e589eb809f 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -477,7 +477,10 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows); - const auto timezone = view.timezone == nullptr ? cctz::utc_time_zone() : *view.timezone; + const auto* source_timezone = view.plain_timestamp_timezone == nullptr + ? view.timezone + : view.plain_timestamp_timezone; + const auto timezone = source_timezone == nullptr ? cctz::utc_time_zone() : *source_timezone; for (size_t row = 0; row < rows; ++row) { const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows); if (orc_serde_utils::orc_row_is_null(batch, source)) { @@ -490,8 +493,7 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, return Status::DataQualityError("Invalid ORC timestamp nanoseconds: {}", nanos); } DateV2Value civil; - const int64_t seconds = - orc_serde_utils::corrected_orc_timestamp_seconds(view, batch.data[source], nanos); + const int64_t seconds = batch.data[source]; auto status = orc_serde_utils::orc_timestamp_to_datetime(seconds, nanos / 1000, timezone, false, &civil); TimeStampNsValue value; diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp b/be/src/core/data_type_serde/orc_serde_utils.cpp index 9dc13722bc2659..e4d78c2b7a2901 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.cpp +++ b/be/src/core/data_type_serde/orc_serde_utils.cpp @@ -36,19 +36,6 @@ bool has_nanosecond_timestamp(const ::orc::Type& type) { return false; } -int64_t corrected_orc_timestamp_seconds(const OrcDecodedColumnView& view, int64_t seconds, - int64_t nanoseconds) { - if (view.writer_epoch_offset == 0 || nanoseconds <= 999999 || - view.file_type->getKind() != ::orc::TIMESTAMP) { - return seconds; - } - // ORC 1.9 applies the negative-fraction correction after changing timezones. Undo it - // in the GMT carrier and apply it at the writer's epoch boundary instead, preserving - // fractions when the timezone shift crosses zero. TIMESTAMP_INSTANT is always UTC. - const int64_t encoded_seconds = seconds < 0 ? seconds + 1 : seconds; - return encoded_seconds < view.writer_epoch_offset ? encoded_seconds - 1 : encoded_seconds; -} - size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { if (selected_rows == nullptr) { return rows; diff --git a/be/src/core/data_type_serde/orc_serde_utils.h b/be/src/core/data_type_serde/orc_serde_utils.h index 14ec35531f1c6a..00a2debe1bb4bb 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.h +++ b/be/src/core/data_type_serde/orc_serde_utils.h @@ -32,9 +32,6 @@ namespace orc_serde_utils { bool has_nanosecond_timestamp(const ::orc::Type& type); -int64_t corrected_orc_timestamp_seconds(const OrcDecodedColumnView& view, int64_t seconds, - int64_t nanoseconds); - size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows); size_t orc_source_row_at(size_t row, const std::vector* selected_rows); bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row); diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index 57ef5a371d1508..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -278,7 +278,6 @@ void OrcFileInputStream::beforeReadStripe( std::unique_ptr<::orc::StripeInformation> current_stripe_information, const std::vector& selected_columns, std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams) { - _writer_timezone = current_stripe_information->getWriterTimezone(); _flush_active_clusters(); _active_stripe_streams.clear(); diff --git a/be/src/format_v2/orc/orc_file_input_stream.h b/be/src/format_v2/orc/orc_file_input_stream.h index 82ea64333ad5b2..3ced728389004f 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.h +++ b/be/src/format_v2/orc/orc_file_input_stream.h @@ -53,7 +53,6 @@ class OrcFileInputStream final : public ::orc::InputStream { uint64_t getNaturalReadSize() const override; void read(void* buf, uint64_t length, uint64_t offset) override; const std::string& getName() const override; - const std::string& writer_timezone() const { return _writer_timezone; } void beforeReadStripe(std::unique_ptr<::orc::StripeInformation> current_stripe_information, const std::vector& selected_columns, @@ -71,7 +70,6 @@ class OrcFileInputStream final : public ::orc::InputStream { std::unordered_map<::orc::StreamId, std::shared_ptr<::orc::InputStream>>& streams); std::string _file_name; - std::string _writer_timezone; io::FileReaderSPtr _file_reader; io::FileReaderSPtr _default_reader; const io::IOContext* _io_ctx = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 6df78c655eca0b..4381e7d37b5730 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -195,6 +195,19 @@ Status set_orc_reader_timezone(const std::string& timezone, return Status::OK(); } +bool has_timestamp(const ::orc::Type& type, bool include_instant) { + if (type.getKind() == ::orc::TIMESTAMP || + (include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) { + return true; + } + for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { + if (has_timestamp(*type.getSubtype(child), include_instant)) { + return true; + } + } + return false; +} + // selected_rows is a source-row remap produced by ORC lazy callback: // predicate columns are decoded first, then surviving row ids drive follower decodes. size_t decode_row_count(size_t rows, const std::vector* selected_rows) { @@ -755,6 +768,8 @@ struct OrcReaderScanState { uint64_t last_stripe = 0; uint64_t offset = 0; uint64_t length = 0; + uint64_t first_row = 0; + uint64_t end_row = 0; }; std::unique_ptr<::orc::Reader> reader; @@ -763,10 +778,8 @@ struct OrcReaderScanState { ::orc::RowReaderOptions row_reader_options; // projection + filter + SARG + stripe range std::string timezone = TimezoneUtils::default_time_zone; cctz::time_zone timezone_obj; - bool utc_wall_clock = false; - OrcFileInputStream* input_stream = nullptr; - std::string stripe_writer_timezone; - int32_t writer_epoch_offset = 0; + bool match_writer_timezone = false; + cctz::time_zone writer_timezone_obj; std::unique_ptr<::orc::RowReader> row_reader; const ::orc::Type* selected_type = nullptr; std::unique_ptr<::orc::ColumnVectorBatch> batch; @@ -791,7 +804,7 @@ struct OrcReaderScanState { std::vector selected_stripe_ranges; size_t current_stripe_range = 0; - bool stripe_pruning_applied = false; + bool stripe_ranges_applied = false; size_t next_row_id = 0; bool row_reader_created = false; @@ -891,7 +904,7 @@ void OrcReader::_collect_profile() const { orc_metric_value(reader_metrics.IOBlockingLatencyUs)); int64_t selected_row_group_count = orc_metric_value(reader_metrics.SelectedRowGroupCount); int64_t evaluated_row_group_count = orc_metric_value(reader_metrics.EvaluatedRowGroupCount); - if (_state->stripe_pruning_applied) { + if (_reader_statistics.filtered_row_groups > 0) { selected_row_group_count = _reader_statistics.read_row_groups; evaluated_row_group_count = _reader_statistics.filtered_row_groups + _reader_statistics.read_row_groups; @@ -955,16 +968,8 @@ Status OrcReader::init(RuntimeState* state) { auto input_stream = std::make_unique( _file_description->path, _file_reader, _io_ctx.get(), _profile, input_stream_options); try { - _state->input_stream = input_stream.get(); _state->reader = ::orc::createReader(std::move(input_stream), options); _state->root_type = &_state->reader->getType(); - if (orc_serde_utils::has_nanosecond_timestamp(*_state->root_type)) { - // Use the same UTC wall-clock carrier as the writer; historical local offsets - // differ between ORC and cctz and can push boundary values out of range. - _state->utc_wall_clock = true; - _state->timezone = "GMT"; - _state->timezone_obj = cctz::utc_time_zone(); - } } catch (const std::exception& e) { if (is_orc_stop(_io_ctx.get(), e)) { return Status::EndOfFile("stop"); @@ -1016,9 +1021,16 @@ void OrcReader::set_condition_cache_context(std::shared_ptrstripe_pruning_applied && _state->selected_stripe_ranges.empty()) { + if (_state->stripe_ranges_applied && _state->selected_stripe_ranges.empty()) { return 0; } + if (_state->match_writer_timezone && _state->stripe_ranges_applied) { + uint64_t rows = 0; + for (const auto& range : _state->selected_stripe_ranges) { + rows += range.end_row - range.first_row; + } + return cast_set(rows); + } if (_state->row_reader != nullptr) { return cast_set(_state->row_reader_range_rows); } @@ -1331,20 +1343,32 @@ Status OrcReader::open(std::shared_ptr request) { _apply_split_range(); RETURN_IF_ERROR(_select_stripe_ranges_by_statistics()); - if (_state->stripe_pruning_applied && _state->selected_stripe_ranges.empty()) { + RETURN_IF_ERROR(_prepare_timestamp_stripe_ranges()); + if (_state->stripe_ranges_applied && _state->selected_stripe_ranges.empty()) { _eof = true; return Status::OK(); } - _apply_current_stripe_range(); + RETURN_IF_ERROR(_apply_current_stripe_range()); RETURN_IF_ERROR(_create_row_reader()); if (_request->row_ids.has_value()) { + size_t range_index = 0; for (const int64_t row_id : *_request->row_ids) { - if (static_cast(row_id) < _state->row_reader_range_first_row || - static_cast(row_id) >= _state->row_reader_range_end_row) { + uint64_t first_row = _state->row_reader_range_first_row; + uint64_t end_row = _state->row_reader_range_end_row; + if (_state->match_writer_timezone) { + const auto& ranges = _state->selected_stripe_ranges; + while (range_index + 1 < ranges.size() && + std::cmp_greater_equal(row_id, ranges[range_index].end_row)) { + ++range_index; + } + first_row = ranges[range_index].first_row; + end_row = ranges[range_index].end_row; + } + if (std::cmp_less(row_id, first_row) || std::cmp_greater_equal(row_id, end_row)) { return Status::InvalidArgument( "ORC row id {} is outside the current split row range [{}, {})", row_id, - _state->row_reader_range_first_row, _state->row_reader_range_end_row); + first_row, end_row); } } } @@ -1402,6 +1426,28 @@ Status OrcReader::_configure_row_reader_projection() { DORIS_CHECK(projection != nullptr); has_complex_projection = has_complex_projection || has_pruned_projection(*projection); } + // A hidden NANOS sibling can affect ordinary timestamps, but scans without civil + // timestamp columns do not need stripe-specific reader timezones. + _state->match_writer_timezone = + orc_serde_utils::has_nanosecond_timestamp(*_state->root_type) && + std::ranges::any_of(_state->read_columns, [&](const auto column_id) { + return !is_virtual_column(column_id) && + has_timestamp(*_state->root_type->getSubtype(column_id.value()), false); + }); + auto filter_projections = _request->predicate_columns; + if (_state->match_writer_timezone && _state->orc_lazy_read_enabled) { + // The pinned SDK's selected timestamp decoder tests a fraction at the wrong row index. + // Decode timestamp leaders densely; keep lazy reads for all other follower columns. + for (const auto column_id : _state->read_columns) { + if (!is_virtual_column(column_id) && + has_timestamp(*_state->root_type->getSubtype(column_id.value()), true) && + std::ranges::none_of(filter_projections, [column_id](const auto& projection) { + return projection.column_id() == column_id; + })) { + filter_projections.push_back(*find_request_projection(*_request, column_id)); + } + } + } if (!has_complex_projection) { std::list include_columns; for (const auto file_column_id : _state->read_columns) { @@ -1413,7 +1459,7 @@ Status OrcReader::_configure_row_reader_projection() { _state->row_reader_options.include(include_columns); if (_state->orc_lazy_read_enabled) { std::list filter_columns; - for (const auto& projection : _request->predicate_columns) { + for (const auto& projection : filter_projections) { if (is_row_position_column(projection.column_id())) { continue; } @@ -1453,8 +1499,8 @@ Status OrcReader::_configure_row_reader_projection() { _state->row_reader_options.includeTypes(include_type_id_list); if (_state->orc_lazy_read_enabled) { std::set filter_type_ids; - RETURN_IF_ERROR(collect_lazy_filter_type_ids( - *_state->root_type, _request->predicate_columns, &filter_type_ids)); + RETURN_IF_ERROR(collect_lazy_filter_type_ids(*_state->root_type, filter_projections, + &filter_type_ids)); DORIS_CHECK(!filter_type_ids.empty()); std::list filter_type_id_list(filter_type_ids.begin(), filter_type_ids.end()); _state->row_reader_options.filterTypes(filter_type_id_list); @@ -1564,7 +1610,7 @@ void OrcReader::_apply_split_range() { Status OrcReader::_select_stripe_ranges_by_statistics() { _state->selected_stripe_ranges.clear(); _state->current_stripe_range = 0; - _state->stripe_pruning_applied = false; + _state->stripe_ranges_applied = false; const bool has_search_argument = _state->row_reader_options.getSearchArgument() != nullptr; if (!has_search_argument) { return Status::OK(); @@ -1623,7 +1669,7 @@ Status OrcReader::_select_stripe_ranges_by_statistics() { return Status::OK(); } - _state->stripe_pruning_applied = true; + _state->stripe_ranges_applied = true; _reader_statistics.filtered_row_groups = cast_set(filtered_stripes); _reader_statistics.filtered_row_groups_by_min_max = cast_set(filtered_stripes); _reader_statistics.filtered_group_rows = filtered_rows; @@ -1669,24 +1715,91 @@ Status OrcReader::_select_stripe_ranges_by_statistics() { return Status::OK(); } -void OrcReader::_apply_current_stripe_range() { - if (!_state->stripe_pruning_applied || _state->selected_stripe_ranges.empty()) { - return; +Status OrcReader::_prepare_timestamp_stripe_ranges() { + if (!_state->match_writer_timezone) { + return Status::OK(); + } + std::vector stripes; + if (_state->stripe_ranges_applied) { + for (const auto& range : _state->selected_stripe_ranges) { + for (auto stripe = range.first_stripe; stripe < range.last_stripe; ++stripe) { + stripes.push_back(stripe); + } + } + } else { + RETURN_IF_ERROR(_collect_split_stripes(&stripes)); + } + _state->selected_stripe_ranges.clear(); + _state->stripe_ranges_applied = true; + _reader_statistics.read_row_groups = cast_set(stripes.size()); + if (stripes.empty()) { + return Status::OK(); + } + try { + uint64_t first_row = 0; + size_t selected = 0; + for (uint64_t index = 0; index <= stripes.back(); ++index) { + const auto stripe = _state->reader->getStripe(index); + const auto end_row = first_row + stripe->getNumberOfRows(); + if (index == stripes[selected]) { + // A RowReader has one timezone, whereas writer timezones belong to stripes. + // Read footer timezone metadata only when entering the selected stripe. + _state->selected_stripe_ranges.push_back({ + .first_stripe = index, + .last_stripe = index + 1, + .offset = stripe->getOffset(), + .length = stripe->getLength(), + .first_row = first_row, + .end_row = end_row, + }); + ++selected; + } + first_row = end_row; + } + } catch (const std::exception& e) { + return Status::InternalError("Failed to prepare ORC timestamp stripe ranges: {}", e.what()); + } + return Status::OK(); +} + +Status OrcReader::_apply_current_stripe_range() { + if (!_state->stripe_ranges_applied || _state->selected_stripe_ranges.empty()) { + return Status::OK(); } DORIS_CHECK(_state->current_stripe_range < _state->selected_stripe_ranges.size()); const auto& stripe_range = _state->selected_stripe_ranges[_state->current_stripe_range]; _state->row_reader_options.range(stripe_range.offset, stripe_range.length); + if (_state->match_writer_timezone) { + try { + const auto stripe = _state->reader->getStripe(stripe_range.first_stripe); + const auto& name = stripe->getWriterTimezone(); + const auto writer_timezone = name.empty() ? "GMT" : name; + if (!TimezoneUtils::find_cctz_time_zone(writer_timezone, _state->writer_timezone_obj)) { + return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", + writer_timezone); + } + // Matching zones preserves the SDK's UTC carrier until its negative-fraction + // correction is complete. cctz then applies historical offsets to that original second. + _state->row_reader_options.setTimezoneName(writer_timezone); + } catch (const std::exception& e) { + if (is_orc_stop(_io_ctx.get(), e)) { + return Status::EndOfFile("stop"); + } + return Status::InternalError("Failed to read ORC stripe writer timezone: {}", e.what()); + } + } + return Status::OK(); } Status OrcReader::_advance_to_next_stripe_range(bool* advanced) { DORIS_CHECK(advanced != nullptr); *advanced = false; - if (!_state->stripe_pruning_applied || _state->selected_stripe_ranges.empty() || + if (!_state->stripe_ranges_applied || _state->selected_stripe_ranges.empty() || _state->current_stripe_range + 1 >= _state->selected_stripe_ranges.size()) { return Status::OK(); } ++_state->current_stripe_range; - _apply_current_stripe_range(); + RETURN_IF_ERROR(_apply_current_stripe_range()); RETURN_IF_ERROR(_create_row_reader()); *advanced = true; return Status::OK(); @@ -1977,19 +2090,8 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type view.selected_rows = selected_rows; view.timezone = &_state->timezone_obj; view.enable_mapping_timestamp_tz = _enable_mapping_timestamp_tz; - if (_state->utc_wall_clock) { - const auto& writer_timezone = _state->input_stream->writer_timezone(); - if (_state->stripe_writer_timezone != writer_timezone) { - cctz::time_zone zone = cctz::utc_time_zone(); - if (!writer_timezone.empty() && - !TimezoneUtils::find_cctz_time_zone(writer_timezone, zone)) { - return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", - writer_timezone); - } - _state->writer_epoch_offset = zone.lookup(cctz::time_point {}).offset; - _state->stripe_writer_timezone = writer_timezone; - } - view.writer_epoch_offset = _state->writer_epoch_offset; + if (_state->match_writer_timezone) { + view.plain_timestamp_timezone = &_state->writer_timezone_obj; } return column_type->get_serde()->read_column_from_orc(*column, view); } @@ -2028,6 +2130,13 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { return Status::OK(); } fetched_row_id = static_cast((*_request->row_ids)[_state->next_row_id]); + // Row ids are sorted globally and can cross the per-writer-timezone ranges. + while (_state->match_writer_timezone && + *fetched_row_id >= _state->row_reader_range_end_row) { + bool advanced = false; + RETURN_IF_ERROR(_advance_to_next_stripe_range(&advanced)); + DORIS_CHECK(advanced); + } _state->row_reader->seekToRow(*fetched_row_id); } // Condition-cache seeks can perform I/O, so keep them in the same cancellation @@ -2160,7 +2269,7 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque } std::vector selected_stripes; - if (_state->stripe_pruning_applied) { + if (_state->stripe_ranges_applied) { for (const auto& stripe_range : _state->selected_stripe_ranges) { if (stripe_range.last_stripe < stripe_range.first_stripe || stripe_range.last_stripe > _state->reader->getNumberOfStripes()) { diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index de8aa40ed70a92..65ad24f7ee40aa 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -130,7 +130,8 @@ class OrcReader final : public format::FileReader { // stripes. Skips the call for a whole-file window to keep ORC library defaults. void _apply_split_range(); Status _select_stripe_ranges_by_statistics(); - void _apply_current_stripe_range(); + Status _prepare_timestamp_stripe_ranges(); + Status _apply_current_stripe_range(); Status _advance_to_next_stripe_range(bool* advanced); Status _create_row_reader(); diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index e4c8a180606b20..1ca11ecb6963e5 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -10579,6 +10579,138 @@ TEST_F(NewOrcReaderTest, NanosecondFilesPreserveNegativeFractionsAcrossWriterTim } } +TEST_F(NewOrcReaderTest, NanosecondFilesPreserveHistoricalWriterCivilTime) { + const auto file_path = (_test_dir / "timestamp_ns_historical.orc").string(); + auto type = ::orc::createStructType(); + type->addStructField("id", ::orc::createPrimitiveType(::orc::INT)); + auto ns_type = ::orc::createPrimitiveType(::orc::TIMESTAMP); + ns_type->setAttribute("iceberg.timestamp-unit", "NANOS"); + type->addStructField("ns", std::move(ns_type)); + type->addStructField("plain", ::orc::createPrimitiveType(::orc::TIMESTAMP)); + type->addStructField("instant", ::orc::createPrimitiveType(::orc::TIMESTAMP_INSTANT)); + MemoryOutputStream stream(1024 * 1024); + ::orc::WriterOptions options; + options.setTimezoneName("America/New_York"); + options.setCompression(::orc::CompressionKind_NONE); + // Flush each tiny batch even while all encoded values still fit in the RLE buffers. + options.setStripeSize(0); + auto writer = ::orc::createWriter(*type, &stream, options); + // The SDK accepts UTC seconds. These cover the historical LMT offset, the second + // preceding a DST fold, and different fractions at the same negative epoch second. + const std::array seconds {-5364644638, -5767201, 1, 1}; + const std::array nanos {0, 500000000, 500000123, 1}; + const std::array expected_ns { + "1800-01-01 00:00:00.000000000", "1969-10-26 01:59:59.500000000", + "1969-12-31 19:00:01.500000123", "1969-12-31 19:00:01.000000001"}; + const std::array expected_plain { + "1800-01-01 00:00:00.000000", "1969-10-26 01:59:59.500000", + "1969-12-31 19:00:01.500000", "1969-12-31 19:00:01.000000"}; + const std::array expected_instant { + "1800-01-01 12:56:02.000000", "1969-10-26 13:59:59.500000", + "1970-01-01 08:00:01.500000", "1970-01-01 08:00:01.000000"}; + for (size_t start = 0; start < seconds.size(); start += 2) { + auto batch = writer->createRowBatch(2); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + root.numElements = 2; + auto& ids = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[0]); + ids.numElements = 2; + for (size_t row = 0; row < 2; ++row) { + ids.data[row] = static_cast(start + row); + for (size_t field = 1; field < root.fields.size(); ++field) { + auto& ts = dynamic_cast<::orc::TimestampVectorBatch&>(*root.fields[field]); + ts.numElements = 2; + ts.data[row] = seconds[start + row]; + ts.nanoseconds[row] = nanos[start + row]; + } + } + writer->add(*batch); + } + writer->close(); + { + std::ofstream out(file_path, std::ios::binary); + out.write(stream.getData(), static_cast(stream.getLength())); + } + ASSERT_EQ(get_orc_stripe_count(file_path), 2); + for (bool project_ns : {false, true}) { + for (int mode : {0, 1, 2, 3}) { + const bool lazy = mode == 1 || mode == 3; + for (int threshold : {-1, 0, 2}) { + auto reader = create_reader_for_path(file_path); + TQueryOptions query_options; + query_options.__set_enable_orc_lazy_mat(lazy); + RuntimeState state {query_options, TQueryGlobals()}; + state.set_timezone("+08:00"); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + if (mode != 3) { + // Isolate the lazy callback from integer stripe pruning, then test both together. + request->metadata_pruning_safe_conjunct_count = 0; + } + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = + project_ns ? decltype(request->non_predicate_columns) {field_projection(1), + field_projection(2)} + : decltype(request->non_predicate_columns) {field_projection(2)}; + request->non_predicate_columns.push_back(field_projection(3)); + request->local_positions = { + {format::LocalColumnId(0), format::LocalIndex(0)}, + {format::LocalColumnId(2), format::LocalIndex(project_ns ? 2 : 1)}}; + request->local_positions.emplace(format::LocalColumnId(3), + format::LocalIndex(project_ns ? 3 : 2)); + if (project_ns) { + request->local_positions.emplace(format::LocalColumnId(1), + format::LocalIndex(1)); + } + if (mode == 2) { + request->predicate_columns.clear(); + request->non_predicate_columns.insert(request->non_predicate_columns.begin(), + field_projection(0)); + request->row_ids = std::vector {}; + for (int row = threshold + 1; row < static_cast(seconds.size()); ++row) { + request->row_ids->push_back(row); + } + } else { + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared(0, threshold))); + } + ASSERT_TRUE(reader->open(request).ok()); + auto block_schema = project_ns ? schema + : std::vector { + schema[0], schema[2], schema[3]}; + Block block = build_file_block(block_schema); + bool eof = false; + size_t output_row = static_cast(threshold + 1); + while (!eof) { + size_t rows = 0; + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + for (size_t row = 0; row < rows; ++row, ++output_row) { + ASSERT_LT(output_row, seconds.size()); + if (project_ns) { + EXPECT_EQ(schema[1].type->to_string(*block.get_by_position(1).column, + row), + expected_ns[output_row]); + } + EXPECT_EQ(schema[2].type->to_string( + *block.get_by_position(project_ns ? 2 : 1).column, row), + expected_plain[output_row]); + // An instant sibling must retain the session zone when plain values use the writer zone. + EXPECT_EQ(schema[3].type->to_string( + *block.get_by_position(project_ns ? 3 : 2).column, row), + expected_instant[output_row]); + } + } + EXPECT_EQ(output_row, seconds.size()); + const auto& stats = reader->reader_statistics(); + EXPECT_EQ(stats.lazy_read_filtered_rows + (lazy ? stats.filtered_group_rows : 0), + lazy ? threshold + 1 : 0); + } + } + } +} + Status decode_orc_timestamp_boundary(int64_t seconds, int64_t nanoseconds, bool file_is_instant, bool use_timestamp_tz, const cctz::time_zone& timezone = cctz::utc_time_zone(), diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java index 1a4b0afb08bb09..58af5063c3e280 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java @@ -133,16 +133,17 @@ public Arrow2PaimonVectorConverter visit(VariantType variantType) { @Override public Arrow2PaimonVectorConverter visit(TimestampType timestampType) { - return timestampAdapter(); + return timestampAdapter(timestampType.getPrecision()); } @Override public Arrow2PaimonVectorConverter visit( LocalZonedTimestampType localZonedTimestampType) { - return timestampAdapter(); + return timestampAdapter(localZonedTimestampType.getPrecision()); } - private static Arrow2PaimonVectorConverter timestampAdapter() { + private static Arrow2PaimonVectorConverter timestampAdapter(int targetPrecision) { + final int nanosPerUnit = (int) Math.pow(10, 9 - targetPrecision); return vector -> { TimeStampVector timestamps = (TimeStampVector) vector; TimeUnit unit = ((ArrowType.Timestamp) vector.getField().getType()).getUnit(); @@ -163,9 +164,12 @@ public Timestamp getTimestamp(int index, int precision) { case MICROSECOND: return Timestamp.fromMicros(value); case NANOSECOND: - return Timestamp.fromEpochMillis( - Math.floorDiv(value, 1_000_000L), - (int) Math.floorMod(value, 1_000_000L)); + // Paimon serializers trust target precision. Truncate the positive + // fractional remainder so negative epochs and leading zero digits + // obey the same narrowing rule, including nested values and keys. + int nanos = (int) Math.floorMod(value, 1_000_000L); + return Timestamp.fromEpochMillis(Math.floorDiv(value, 1_000_000L), + nanos / nanosPerUnit * nanosPerUnit); default: throw new IllegalArgumentException( "Unsupported Paimon Arrow timestamp unit: " + unit); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java index daaa2bb45ce1e4..1285c4f25007e0 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java @@ -40,14 +40,25 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.data.variant.Variant; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.CloseableIterator; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.ByteArrayInputStream; import java.math.BigDecimal; +import java.nio.file.Path; import java.time.ZoneId; import java.util.Arrays; import java.util.LinkedHashMap; @@ -209,6 +220,91 @@ public void testTimestampAdaptationDoesNotRepeatTimezoneConversion() throws Exce } } + @Test + public void testNanosecondTransportRespectsTargetPrecision() throws Exception { + for (int precision : new int[] {7, 8, 9}) { + RowType inputType = DataTypes.ROW(DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(precision)), + DataTypes.FIELD(1, "nested", DataTypes.ROW( + DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(precision))))); + try (RootAllocator allocator = new RootAllocator()) { + PaimonArrowBatchAdapter adapter = new PaimonArrowBatchAdapter(inputType, ZoneId.of("UTC"), allocator); + try (ArrowStreamReader reader = schemaReader(adapter, allocator)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + root.allocateNew(); + StructVector nested = (StructVector) root.getVector("nested"); + for (long nanos : new long[] {123456789, -123456789, -1, 0, 1}) { + ((TimeStampVector) root.getVector("ts")).setSafe(0, nanos); + ((TimeStampVector) nested.getChild("ts")).setSafe(0, nanos); + nested.setIndexDefined(0); + root.setRowCount(1); + InternalRow row = adapter.rows(root).row(0); + long unit = precision == 7 ? 100 : precision == 8 ? 10 : 1; + long expected = Math.floorDiv(nanos, unit) * unit; + for (org.apache.paimon.data.Timestamp value : new org.apache.paimon.data.Timestamp[] { + row.getTimestamp(0, precision), row.getRow(1, 1).getTimestamp(0, precision)}) { + Assertions.assertEquals(Math.floorDiv(expected, 1_000_000L), value.getMillisecond()); + Assertions.assertEquals(Math.floorMod(expected, 1_000_000L), value.getNanoOfMillisecond()); + } + } + } + } + } + } + + @Test + public void testPrecisionSevenAndEightKeysRoundTrip(@TempDir Path dir) + throws Exception { + for (String format : new String[] {"parquet", "orc"}) { + for (int precision : new int[] {7, 8}) { + LocalFileIO fileIO = LocalFileIO.create(); + org.apache.paimon.fs.Path path = new org.apache.paimon.fs.Path( + dir.resolve(format + precision).toUri()); + org.apache.paimon.schema.Schema schema = org.apache.paimon.schema.Schema.newBuilder() + .column("ts", DataTypes.TIMESTAMP(precision).notNull()) + .column("detail", DataTypes.ROW(DataTypes.FIELD(0, "t", DataTypes.TIMESTAMP(precision)))) + .primaryKey("ts").option("bucket", "1").option("file.format", format).build(); + new SchemaManager(fileIO, path).createTable(schema); + Table table = FileStoreTableFactory.create(fileIO, path); + PaimonWriteSchema writeSchema = PaimonWriteSchema.create(table.rowType(), new String[] {"ts", "detail"}); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (RootAllocator allocator = new RootAllocator(); + BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + PaimonArrowBatchAdapter adapter = new PaimonArrowBatchAdapter( + table.rowType(), ZoneId.of("UTC"), allocator); + try (ArrowStreamReader reader = schemaReader(adapter, allocator)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + root.allocateNew(); + for (long nanos : new long[] {123456781, 123456789}) { + ((TimeStampVector) root.getVector("ts")).setSafe(0, nanos); + StructVector nested = (StructVector) root.getVector("detail"); + ((TimeStampVector) nested.getChild("t")).setSafe(0, nanos); + nested.setIndexDefined(0); + root.setRowCount(1); + write.write(writeSchema.tableRow(adapter.rows(root).row(0))); + } + } + commit.commit(write.prepareCommit()); + } + ReadBuilder read = table.newReadBuilder(); + int count = 0; + try (CloseableIterator rows = + read.newRead().createReader(read.newScan().plan()).toCloseableIterator()) { + while (rows.hasNext()) { + InternalRow row = rows.next(); + int expected = precision == 7 ? 456700 : 456780; + Assertions.assertEquals(expected, row.getTimestamp(0, precision).getNanoOfMillisecond()); + Assertions.assertEquals(expected, + row.getRow(1, 1).getTimestamp(0, precision).getNanoOfMillisecond()); + count++; + } + } + // Precision narrowing must happen before hashing and serialization of primary keys. + Assertions.assertEquals(1, count); + } + } + } + @Test public void testNanosecondTransportPreservesSignedBounds() throws Exception { RowType inputType = DataTypes.ROW(DataTypes.FIELD(0, "ts", DataTypes.TIMESTAMP(9))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index ae06f269cca17b..f7a6969d2ba3e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -28,6 +28,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -559,6 +560,13 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c try { ColumnDef.validateDefaultValue(type.toCatalogDataType(), defaultValue.get().getValue(), defaultValue.get().getDefaultValueExprDef()); + if (!isOlap && type.isTimeStampNsType() + && defaultValue.get().getDefaultValueExprDef() == null) { + // Connectors receive civil SQL defaults, not numeric epoch carriers. Normalize + // with the same parser as validation so compact dates cannot become epoch nanos. + defaultValue = Optional.of(new DefaultValue( + new TimeStampNsLiteral(defaultValue.get().getValue()).getStringValue())); + } } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index 5a919c81a6d1e2..9285d7027a8c34 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.QuantileStateType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.UuidType; import com.google.common.collect.ImmutableList; @@ -50,6 +51,22 @@ public void tearDown() { Config.enable_non_aggregate_table_state_types = false; } + @Test + public void testTimestampNsDefaultsAreNormalizedForExternalDdl() { + String[][] values = { + {"20240102", "2024-01-02 00:00:00.000000000"}, + {"20240102030405", "2024-01-02 03:04:05.000000000"}, + {"2024-01-02 03:04:05.123456789", "2024-01-02 03:04:05.123456789"} + }; + for (String[] value : values) { + ColumnDefinition column = new ColumnDefinition("ts", TimeStampNsType.INSTANCE, + false, null, true, Optional.of(new DefaultValue(value[0])), ""); + column.validate(false, ImmutableSet.of(), ImmutableSet.of(), false, KeysType.DUP_KEYS); + Assertions.assertEquals(value[1], column.translateToCatalogStyle().getDefaultValue()); + Assertions.assertEquals(value[1], column.translateToCatalogStyleForSchemaChange().getDefaultValue()); + } + } + @Test public void testNameEquals() { ColumnDefinition columnDefinition = new ColumnDefinition("col1", null, false, null, false, null, null); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out index f3a38239c85848..6e170b9882bf71 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -61,6 +61,11 @@ entries map Yes true \N 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 +-- !parquet_compact_defaults -- +1 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 +2 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 +3 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 + -- !parquet_identity -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 @@ -229,6 +234,11 @@ entries map Yes true \N 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 +-- !orc_compact_defaults -- +1 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 +2 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 +3 2024-01-02 00:00:00.000000000 2024-01-02 03:04:05.000000000 + -- !orc_identity -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy index 4e003f850133a5..6a06d496e46597 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -85,6 +85,13 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { sql "INSERT INTO ${defaultsTable} (id) VALUES (1)" sql "INSERT INTO ${defaultsTable} VALUES (2, '2024-01-02 03:04:05.123456788')" "qt_${format}_defaults" "SELECT id, CAST(ts AS STRING) FROM ${defaultsTable} ORDER BY id" + // Compact SQL literals must persist the same initial and write defaults as canonical text. + sql """ALTER TABLE ${defaultsTable} ADD COLUMN compact_date TIMESTAMP_NS DEFAULT '20240102'""" + sql """ALTER TABLE ${defaultsTable} ADD COLUMN compact_datetime TIMESTAMP_NS + DEFAULT '20240102030405'""" + sql "INSERT INTO ${defaultsTable} (id) VALUES (3)" + "qt_${format}_compact_defaults" """SELECT id, CAST(compact_date AS STRING), + CAST(compact_datetime AS STRING) FROM ${defaultsTable} ORDER BY id""" for (String transform : ["identity", "year", "month", "day", "hour", "bucket"]) { String partitionTable = "${table}_${transform}" String expression = transform == "identity" ? "ts" From 32f59f94281ebdd61451106031d87c7e1e2edbd2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 09:39:04 +0800 Subject: [PATCH 07/13] [fix](external) Check Iceberg date promotions and initialize ORC test timezones ### What problem does this PR solve? Related PR: #68934 Problem Summary: Retained Iceberg DATE files keep epoch-day manifest bounds after promotion to timestamp_ns. Iceberg's metrics evaluator interprets those four-byte bounds as unscaled longs and can prune matching files. Suppress unsafe predicates for field IDs with historical DATE definitions across SDK, cached, conflict and rewrite planning, while retaining pushdown for native nanosecond fields. Force checked DATE-to-nanosecond schema conversions, including nested values and equality keys, so overflow cannot silently become NULL. Initialize the timezone cache in the ORC writer test fixture to make isolated test execution deterministic. ### Release note Fix missing rows and silent NULL values when reading Iceberg DATE fields promoted to timestamp_ns. ### Check List (For Author) - Test: 323 related FE tests; 386 related BE tests; FE Checkstyle, clang-format 16, changed-line clang-tidy and BE build hygiene. - Behavior changed: Yes, retained DATE files are not incorrectly pruned and out-of-range promotions fail regardless of session strict-cast mode. - Does this need documentation: No --- be/src/format_v2/column_mapper.cpp | 2 +- be/src/format_v2/expr/cast.cpp | 8 ++ be/src/format_v2/expr/cast.h | 6 +- be/src/format_v2/table/iceberg_reader.cpp | 13 ++-- be/src/format_v2/table_reader.h | 2 +- .../transformer/vorc_transformer_test.cpp | 5 +- .../format_v2/table/iceberg_reader_test.cpp | 77 +++++++++++++++++++ .../iceberg/IcebergConnectorTransaction.java | 3 +- .../iceberg/IcebergPredicateConverter.java | 25 +++++- .../iceberg/IcebergScanPlanProvider.java | 21 +++-- .../rewrite/RewriteDataFilePlanner.java | 5 +- .../iceberg/IcebergScanPlanProviderTest.java | 46 +++++++++++ 12 files changed, 189 insertions(+), 24 deletions(-) diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index c991b2fd940a61..ae75b6f15cb1af 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2189,7 +2189,7 @@ static void rebuild_projection(ColumnMapping* mapping, LocalIndex block_position return; } - auto expr = Cast::create_shared(mapping->table_type); + auto expr = Cast::create_shared(mapping->table_type, true); expr->add_child(VSlotRef::create_shared(cast_set(block_position.value()), cast_set(block_position.value()), -1, mapping->file_type, mapping->file_column_name)); diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index c1435732a9e136..8c529f18b2fa3b 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -27,6 +27,7 @@ #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/block/columns_with_type_and_name.h" +#include "core/data_type/data_type_nullable.h" #include "exprs/function/simple_function_factory.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" @@ -65,6 +66,13 @@ Status Cast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContex return Status::InternalError("Could not find function {} ", _expr_name); } VExpr::register_function_context(state, context); + if (_is_schema_evolution && + remove_nullable(_children[0]->data_type())->get_primitive_type() == TYPE_DATEV2 && + remove_nullable(_data_type)->get_primitive_type() == TYPE_TIMESTAMP_NS) { + // A retained DATE file must fail outside the promoted nanosecond range, never synthesize NULL. + // Keep this independent of SQL CAST semantics and the session's strict-cast setting. + context->fn_context(_fn_context_index)->set_enable_strict_mode(true); + } _prepare_finished = true; return Status::OK(); } diff --git a/be/src/format_v2/expr/cast.h b/be/src/format_v2/expr/cast.h index 18b3bea676c4fc..07f54bb29d7d28 100644 --- a/be/src/format_v2/expr/cast.h +++ b/be/src/format_v2/expr/cast.h @@ -38,7 +38,8 @@ class Cast final : public VExpr { ENABLE_FACTORY_CREATOR(Cast); public: - Cast(const DataTypePtr& type) { + Cast(const DataTypePtr& type, bool is_schema_evolution = false) + : _is_schema_evolution(is_schema_evolution) { _node_type = TExprNodeType::CAST_EXPR; _opcode = TExprOpcode::CAST; _data_type = type; @@ -58,11 +59,12 @@ class Cast final : public VExpr { bool is_safe_to_execute_on_selected_rows() const override { return false; } Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); - *cloned_expr = Cast::create_shared(_data_type); + *cloned_expr = Cast::create_shared(_data_type, _is_schema_evolution); return Status::OK(); } private: + bool _is_schema_evolution; Status _do_execute(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const; std::string _expr_name; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 6af63be1374e0a..b33265259c4a5b 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -653,7 +653,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit *key_expr = std::move(literal); return Status::OK(); } - auto cast_expr = Cast::create_shared(delete_key_type); + auto cast_expr = Cast::create_shared(delete_key_type, true); cast_expr->add_child(literal); *key_expr = std::move(cast_expr); return Status::OK(); @@ -1511,7 +1511,7 @@ Status IcebergTableReader::_build_missing_equality_delete_key_expr( DORIS_CHECK(table_leaf != nullptr); DORIS_CHECK(table_leaf->type != nullptr); if (!table_leaf->type->equals(*filter.key_types[key_idx])) { - auto cast_expr = Cast::create_shared(filter.key_types[key_idx]); + auto cast_expr = Cast::create_shared(filter.key_types[key_idx], true); cast_expr->add_child(default_expr); default_expr = std::move(cast_expr); } @@ -1565,7 +1565,7 @@ Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe if (field->type->equals(*filter.key_types[idx])) { delete_predicate->add_child(key_expr); } else { - auto cast_expr = Cast::create_shared(filter.key_types[idx]); + auto cast_expr = Cast::create_shared(filter.key_types[idx], true); cast_expr->add_child(key_expr); delete_predicate->add_child(cast_expr); } @@ -1832,10 +1832,9 @@ Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDe path, request->local_positions.at(root_column_id).value(), &key_expr)); const auto& key_type = result->key_types[index]; if (!key_expr->data_type()->equals(*key_type)) { - // Historical delete values are promoted into the comparison domain. For an - // Iceberg-legal type promotion this cast is always widening, so it cannot turn a - // non-NULL value into NULL. - auto cast_expr = Cast::create_shared(key_type); + // Historical delete values share the promoted comparison domain. DATE-to-nanos + // must report overflow rather than turn a non-NULL equality key into NULL. + auto cast_expr = Cast::create_shared(key_type, true); cast_expr->add_child(key_expr); key_expr = std::move(cast_expr); } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 7aa4400320f50e..f8b1d529a473e8 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1357,7 +1357,7 @@ class TableReader { Block cast_block; cast_block.insert({*column, input_type, column_name}); auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name); - auto cast_expr = Cast::create_shared(cast_type); + auto cast_expr = Cast::create_shared(cast_type, true); cast_expr->add_child(std::move(slot_ref)); auto cast_ctx = VExprContext::create_shared(std::move(cast_expr)); RowDescriptor row_desc; diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp index 0b933ba16ec2c5..5eeeef7fa45767 100644 --- a/be/test/format/transformer/vorc_transformer_test.cpp +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -56,6 +56,8 @@ namespace doris { class VOrcTransformerTest : public testing::Test { protected: void SetUp() override { + // Writers use the startup timezone cache; each test must work without another suite initializing it. + TimezoneUtils::load_timezones_to_cache(); _file_path = "./vorc_transformer_" + UniqueId::gen_uid().to_string() + ".orc"; _fs = io::global_local_filesystem(); } @@ -111,7 +113,8 @@ TEST_F(VOrcTransformerTest, NanosecondBoundsUseUtcWallClockCarrier) { for (auto value : nanos) { column->insert(Field::create_field(TimeStampNsValue(value))); } - ASSERT_TRUE(writer.write(Block({{std::move(column), type, "ts"}})).ok()); + const auto write_status = writer.write(Block({{std::move(column), type, "ts"}})); + ASSERT_TRUE(write_status.ok()) << write_status; ASSERT_TRUE(writer.close().ok()); TIcebergColumnStats stats; ASSERT_TRUE(writer.collect_file_statistics_after_close(&stats).ok()); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 8ee7788a40e60a..749624c5fa456f 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -58,6 +58,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_uuid.h" #include "core/data_type/data_type_varbinary.h" @@ -1665,6 +1666,82 @@ void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { ASSERT_TRUE(status.ok()) << status; } +TEST(IcebergV2ReaderTest, DatePromotionToNanosecondsChecksOverflow) { + const auto test_dir = std::filesystem::temp_directory_path() / "iceberg_date_promotion"; + std::filesystem::create_directories(test_dir); + for (bool nested : {false, true}) { + for (int32_t days : {19724, -354285}) { + const auto file_path = + (test_dir / (std::to_string(days) + (nested ? "_nested.parquet" : ".parquet"))) + .string(); + // Read an actual retained DATE file, including NULL, through field-ID schema evolution. + auto field = arrow::field("ts", arrow::date32(), true, + arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + arrow::Date32Builder builder; + ASSERT_TRUE(builder.Append(days).ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr values = builder.Finish().ValueOrDie(); + if (nested) { + values = arrow::StructArray::Make({values}, {field}).ValueOrDie(); + field = arrow::field("outer", values->type(), false, + arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); + } + auto out = arrow::io::FileOutputStream::Open(file_path).ValueOrDie(); + ASSERT_TRUE(::parquet::arrow::WriteTable( + *arrow::Table::Make(arrow::schema({field}), {values}), + arrow::default_memory_pool(), out, 2) + .ok()); + ASSERT_TRUE(out->Close().ok()); + for (bool strict : {false, true}) { + SCOPED_TRACE(testing::Message() + << "nested=" << nested << " days=" << days << " strict=" << strict); + TQueryOptions query_options; + query_options.__set_enable_strict_cast(strict); + RuntimeState state {query_options, TQueryGlobals()}; + const auto type = make_nullable(std::make_shared()); + std::vector columns {make_table_column(1, "ts", type)}; + if (nested) { + auto outer = make_table_column( + 2, "outer", + std::make_shared(DataTypes {type}, Strings {"ts"})); + outer.children = columns; + columns = {outer}; + } + auto scan_params = make_local_parquet_scan_params(); + RuntimeProfile profile("date_promotion"); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, columns, &scan_params, io_ctx, &state, &profile); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + Block block = build_table_block(columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + if (days < 0) { + EXPECT_FALSE(status.ok()) + << "out-of-range promotion must fail even in non-strict mode"; + EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), std::string::npos) + << status; + } else { + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(block.rows(), 2); + const IColumn* result = block.get_by_position(0).column.get(); + if (nested) { + result = &assert_cast( + expect_not_null_table_column(block, 0)) + .get_column(0); + } + EXPECT_EQ(type->to_string(*result, 0), "2024-01-02 00:00:00.000000000"); + EXPECT_TRUE(result->is_null_at(1)); + } + ASSERT_TRUE(reader.close().ok()); + } + } + } + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, AnnotateBuildsTypedNestedInitialDefault) { const auto int_type = make_nullable(std::make_shared()); const auto struct_type = make_nullable( diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index f1ea0923672406..8f38b3186062ed 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -1140,7 +1140,8 @@ Optional buildWriteConstraintExpression(Table icebergTable) { // matrix is a conservative port of legacy convertPredicateToIcebergExpression (IS NULL / BETWEEN / // same-column OR / NOT(IS NULL); drops NE / cross-column OR) — different from scan pushdown. List converted = - new IcebergPredicateConverter(icebergTable.schema(), zone, true).convert(expr); + IcebergPredicateConverter.forTable(icebergTable, icebergTable.schema(), zone, + IcebergPredicateConverter.Mode.CONFLICT).convert(expr); if (converted.isEmpty()) { return Optional.empty(); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java index b0a96fae59864e..6c6ae32f18138f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java @@ -29,6 +29,7 @@ import org.apache.doris.connector.spi.pushdown.ConnectorOr; import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -38,6 +39,7 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Type.TypeID; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -104,6 +106,7 @@ public class IcebergPredicateConverter { private static final List SIGNED_ZEROS = Collections.unmodifiableList(Arrays.asList(-0.0d, 0.0d)); private final Schema schema; + private final Set promotedDateFields = new LinkedHashSet<>(); private final ZoneId sessionZone; // The conversion matrix. SCAN = scan-time pushdown (BE re-filters, so widening is safe). CONFLICT = O5-2 // write-time optimistic conflict detection (no-missed-conflict, so widening is also safe) -- a port of @@ -130,6 +133,23 @@ public IcebergPredicateConverter(Schema schema, ZoneId sessionZone, Mode mode) { this.mode = mode; } + public static IcebergPredicateConverter forTable(Table table, Schema schema, ZoneId zone, Mode mode) { + IcebergPredicateConverter converter = new IcebergPredicateConverter(schema, zone, mode); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.type().typeId() != TypeID.TIMESTAMP_NANO) { + continue; + } + for (Schema historical : table.schemas().values()) { + Type oldType = historical.findType(field.fieldId()); + if (oldType != null && oldType.typeId() == TypeID.DATE) { + converter.promotedDateFields.add(field.fieldId()); + break; + } + } + } + return converter; + } + /** * Convert a {@link ConnectorExpression} tree into a list of iceberg {@link Expression}s. Top-level AND * nodes are flattened into the list and each top-level conjunct is built + bind-checked independently — @@ -476,7 +496,10 @@ private Types.NestedField getPushdownField(String colName) { || ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL.equalsIgnoreCase(colName)) { return null; } - return schema.caseInsensitiveFindField(colName); + Types.NestedField field = schema.caseInsensitiveFindField(colName); + // Iceberg 1.11 widens retained four-byte DATE bounds to unscaled longs. Pruning those + // epoch-day bounds with nanosecond predicates can discard matching files in every planner. + return field != null && promotedDateFields.contains(field.fieldId()) ? null : field; } /** diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index 8ad3d1cc2bf5be..b612be4f42d9d2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -1050,8 +1050,9 @@ private List doPlanPositionDeletesSystemTableScan(IcebergTab } } if (filter.isPresent()) { - List predicates = new IcebergPredicateConverter( - metadataTable.schema(), resolveSessionZone(session)).convert(filter.get()); + List predicates = IcebergPredicateConverter.forTable( + metadataTable, metadataTable.schema(), resolveSessionZone(session), + IcebergPredicateConverter.Mode.SCAN).convert(filter.get()); for (Expression predicate : predicates) { scan = scan.filter(predicate); } @@ -1264,7 +1265,8 @@ private TableScan buildScan(Table table, IcebergTableHandle handle, Optional predicates = - new IcebergPredicateConverter(predicateSchema, resolveSessionZone(session)).convert(filter.get()); + IcebergPredicateConverter.forTable(table, predicateSchema, resolveSessionZone(session), + IcebergPredicateConverter.Mode.SCAN).convert(filter.get()); for (Expression predicate : predicates) { scan = scan.filter(predicate); } @@ -2129,8 +2131,9 @@ public Map getScanNodeProperties( // IcebergScanNode `if (!pushdownIcebergPredicates.isEmpty())`). if (!systemTable && filter.isPresent()) { Schema predicateSchema = iceHandle.hasSnapshotPin() ? pinnedSchema(table, iceHandle) : table.schema(); - List pushed = new IcebergPredicateConverter( - predicateSchema, resolveSessionZone(session)).convert(filter.get()); + List pushed = IcebergPredicateConverter.forTable( + table, predicateSchema, resolveSessionZone(session), + IcebergPredicateConverter.Mode.SCAN).convert(filter.get()); if (!pushed.isEmpty()) { StringBuilder sb = new StringBuilder(); for (Expression predicate : pushed) { @@ -2811,7 +2814,7 @@ private CloseableIterable cacheBackedFileScanTasks(TableScan scan, return CloseableIterable.withNoopClose(Collections.emptyList()); } Schema scanSchema = scan.schema(); - Expression filterExpr = combineFilter(filter, scanSchema, session); + Expression filterExpr = combineFilter(filter, table, scanSchema, session); Map specsById = SchemaAwareDataTableScan.specsFor(scan); boolean caseSensitive = true; @@ -2969,12 +2972,14 @@ public void close() throws IOException { * the T02 {@link IcebergPredicateConverter} on the scan-bound schema; an absent filter is * {@code alwaysTrue()} (scan everything). */ - private Expression combineFilter(Optional filter, Schema schema, ConnectorSession session) { + private Expression combineFilter(Optional filter, Table table, + Schema schema, ConnectorSession session) { if (!filter.isPresent()) { return Expressions.alwaysTrue(); } List predicates = - new IcebergPredicateConverter(schema, resolveSessionZone(session)).convert(filter.get()); + IcebergPredicateConverter.forTable(table, schema, resolveSessionZone(session), + IcebergPredicateConverter.Mode.SCAN).convert(filter.get()); Expression combined = Expressions.alwaysTrue(); for (Expression predicate : predicates) { combined = Expressions.and(combined, predicate); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlanner.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlanner.java index b4cd7f51b6b312..4a51d43ace932f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlanner.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/rewrite/RewriteDataFilePlanner.java @@ -123,8 +123,9 @@ CloseableIterable planFileScanTasks(Table icebergTable) { // the legacy live-rewrite behaviour, which threw, with master's full matrix). if (parameters.hasWhereCondition()) { ConnectorExpression where = parameters.getWhereCondition().getExpression(); - List predicates = new IcebergPredicateConverter( - icebergTable.schema(), sessionZone, IcebergPredicateConverter.Mode.REWRITE).convert(where); + List predicates = IcebergPredicateConverter.forTable( + icebergTable, icebergTable.schema(), sessionZone, + IcebergPredicateConverter.Mode.REWRITE).convert(where); if (predicates.size() < countTopLevelConjuncts(where)) { throw new DorisConnectorException( "WHERE condition for rewrite_data_files cannot be pushed down to file pruning: " + where); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 8654cc569de1a4..9344a9d961e0e7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -4045,6 +4045,52 @@ public void manifestCacheUsesLatestSchemaForSchemaOnlyMvccPin() throws IOExcepti Assertions.assertEquals(0L, cache.takeStats("q")[2], "the cache path must not fall back to the SDK"); } + @Test + public void promotedDateBoundsDoNotPruneNanosecondMatches() { + Schema oldSchema = new Schema(Types.NestedField.optional(1, "ts", Types.DateType.get())); + Table table = createTable("promoted_date", oldSchema, PartitionSpec.unpartitioned(), + Collections.singletonMap(TableProperties.FORMAT_VERSION, "3")); + Map bounds = Collections.singletonMap( + 1, Conversions.toByteBuffer(Types.DateType.get(), 19724)); + table.newAppend().appendFile(DataFiles.builder(table.spec()) + .withPath("/data/retained-date.parquet").withFileSizeInBytes(100).withRecordCount(1) + .withMetrics(new Metrics(1L, null, null, null, null, bounds, bounds)) + .withFormat(FileFormat.PARQUET).build()).commit(); + org.apache.iceberg.TableOperations operations = ((org.apache.iceberg.HasTableOperations) table).operations(); + org.apache.iceberg.TableMetadata base = operations.current(); + // Publish the v3 promotion directly: retained manifests still encode DATE bounds in days. + operations.commit(base, org.apache.iceberg.TableMetadata.buildFrom(base) + .setCurrentSchema(new Schema( + Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone()), + Types.NestedField.optional(2, "fresh", Types.TimestampNanoType.withoutZone())), 2).build()); + table.refresh(); + ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), + new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-02 00:00:00")); + for (IcebergPredicateConverter.Mode mode : IcebergPredicateConverter.Mode.values()) { + IcebergPredicateConverter converter = IcebergPredicateConverter.forTable( + table, table.schema(), java.time.ZoneOffset.UTC, mode); + Assertions.assertTrue(converter.convert(filter).isEmpty()); + Assertions.assertEquals(1, converter.convert(new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("fresh", ConnectorType.of("TIMESTAMP_NS")), + new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-02 00:00:00"))).size(), + "fields without historical DATE bounds must retain pushdown"); + } + ConnectorScanRequest request = ConnectorScanRequest.builder( + new IcebergTableHandle("db1", "promoted_date"), Collections.emptyList()) + .filter(Optional.of(filter)).build(); + List sdk = new IcebergScanPlanProvider( + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) + .planScan(emptySession(), request); + Assertions.assertEquals(1, sdk.size(), "SDK planning must preserve the matching DATE file"); + IcebergManifestCache cache = new IcebergManifestCache(); + List cached = manifestProvider(manifestCacheProps(), table, cache) + .planScan(emptySession(), request); + Assertions.assertEquals(sortedPaths(sdk), sortedPaths(cached)); + Assertions.assertTrue(cache.size() > 0); + Assertions.assertEquals(0L, cache.takeStats("q")[2], "must exercise cached planning without fallback"); + } + // --- T09: vended credentials (extractVendedToken + static/vended location.* + URI threading) --- @Test From ba7341cbdcb594efd24a1348ec508fa1b7b33cd6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 10:38:56 +0800 Subject: [PATCH 08/13] [fix](iceberg) Preserve nanosecond semantics after cleanup and filtering ### What problem does this PR solve? Related PR: #68934 Snapshot expiration can remove a DATE writing schema while retaining its data files. Treat incomplete schema history as unsafe for nanosecond predicate pushdown so day bounds cannot prune matching files. Preserve existing pushdown when complete history proves a field has no DATE promotion. Mark file-local casts as schema evolution so IS NOT NULL cannot hide DATE to TIMESTAMP_NS overflow in non-strict sessions. Interpret explicit offsets in nanosecond static partition values using the sink's session-local semantics, so overwrite deletes the partition that receives the replacement rows. ### Release note Fix missing Iceberg rows after schema metadata cleanup, filtered DATE promotion overflow, and incorrect partition replacement for offset timestamp literals. ### Check List (For Author) - Test: FE unit tests, BE unit tests, and Iceberg Parquet/ORC regression tests - Behavior changed: Yes; preserve query errors and target the correct overwrite partition - Does this need documentation: No --- be/src/format_v2/column_mapper.cpp | 4 +- .../format_v2/table/iceberg_reader_test.cpp | 104 +++++++++++------- .../iceberg/IcebergPartitionUtils.java | 15 ++- .../iceberg/IcebergPredicateConverter.java | 19 +++- .../IcebergConnectorTransactionTest.java | 27 +++++ .../iceberg/IcebergScanPlanProviderTest.java | 16 ++- .../iceberg/IcebergTimestampNsTest.java | 15 +++ .../iceberg/test_iceberg_timestamp_ns.out | 22 ++++ .../iceberg/test_iceberg_timestamp_ns.groovy | 7 ++ 9 files changed, 180 insertions(+), 49 deletions(-) diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index ae75b6f15cb1af..02b08f450e3f4e 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -1102,7 +1102,9 @@ static VExprSPtr cast_file_expr_to_table_type(const VExprSPtr& file_expr, DORIS_CHECK(file_expr != nullptr); DORIS_CHECK(table_type != nullptr); DORIS_CHECK(rewrite_context != nullptr); - auto cast_expr = Cast::create_shared(table_type); + // Localized filters perform the same evolution as projection; SQL non-strict casts must + // not replace an overflowing retained DATE with NULL and filter it out before materialization. + auto cast_expr = Cast::create_shared(table_type, true); cast_expr->add_child(file_expr); rewrite_context->add_created_expr(cast_expr); return cast_expr; diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 749624c5fa456f..5de1919f9be026 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -1692,50 +1692,74 @@ TEST(IcebergV2ReaderTest, DatePromotionToNanosecondsChecksOverflow) { arrow::default_memory_pool(), out, 2) .ok()); ASSERT_TRUE(out->Close().ok()); - for (bool strict : {false, true}) { - SCOPED_TRACE(testing::Message() - << "nested=" << nested << " days=" << days << " strict=" << strict); - TQueryOptions query_options; - query_options.__set_enable_strict_cast(strict); - RuntimeState state {query_options, TQueryGlobals()}; - const auto type = make_nullable(std::make_shared()); - std::vector columns {make_table_column(1, "ts", type)}; - if (nested) { - auto outer = make_table_column( - 2, "outer", - std::make_shared(DataTypes {type}, Strings {"ts"})); - outer.children = columns; - columns = {outer}; + for (bool filtered : {false, true}) { + if (nested && filtered) { + continue; } - auto scan_params = make_local_parquet_scan_params(); - RuntimeProfile profile("date_promotion"); - io::FileReaderStats file_reader_stats; - io::FileCacheStatistics file_cache_stats; - auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); - doris::format::iceberg::IcebergTableReader reader; - init_iceberg_reader(&reader, columns, &scan_params, io_ctx, &state, &profile); - ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); - Block block = build_table_block(columns); - bool eos = false; - const auto status = reader.get_block(&block, &eos); - if (days < 0) { - EXPECT_FALSE(status.ok()) - << "out-of-range promotion must fail even in non-strict mode"; - EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), std::string::npos) - << status; - } else { - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(block.rows(), 2); - const IColumn* result = block.get_by_position(0).column.get(); + for (bool strict : {false, true}) { + SCOPED_TRACE(testing::Message() + << "nested=" << nested << " days=" << days << " strict=" << strict + << " filtered=" << filtered); + TQueryOptions query_options; + query_options.__set_enable_strict_cast(strict); + RuntimeState state {query_options, TQueryGlobals()}; + const auto type = make_nullable(std::make_shared()); + std::vector columns {make_table_column(1, "ts", type)}; if (nested) { - result = &assert_cast( - expect_not_null_table_column(block, 0)) - .get_column(0); + auto outer = make_table_column( + 2, "outer", + std::make_shared(DataTypes {type}, Strings {"ts"})); + outer.children = columns; + columns = {outer}; + } + auto scan_params = make_local_parquet_scan_params(); + RuntimeProfile profile("date_promotion"); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + doris::format::iceberg::IcebergTableReader reader; + VExprContextSPtrs conjuncts; + if (filtered) { + // A file-local filter must not hide overflow before strict table materialization. + auto predicate = table_function_expr( + "is_not_null_pred", std::make_shared(), {type}); + predicate->add_child(VSlotRef::create_shared(0, 0, 0, type, "ts")); + conjuncts.push_back(prepared_conjunct(&state, predicate)); } - EXPECT_EQ(type->to_string(*result, 0), "2024-01-02 00:00:00.000000000"); - EXPECT_TRUE(result->is_null_at(1)); + ASSERT_TRUE(reader.init({.projected_columns = columns, + .conjuncts = conjuncts, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + Block block = build_table_block(columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + if (days < 0) { + EXPECT_FALSE(status.ok()) + << "out-of-range promotion must fail even in non-strict mode"; + EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), + std::string::npos) + << status; + } else { + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(block.rows(), filtered ? 1 : 2); + const IColumn* result = block.get_by_position(0).column.get(); + if (nested) { + result = &assert_cast( + expect_not_null_table_column(block, 0)) + .get_column(0); + } + EXPECT_EQ(type->to_string(*result, 0), "2024-01-02 00:00:00.000000000"); + if (!filtered) { + EXPECT_TRUE(result->is_null_at(1)); + } + } + ASSERT_TRUE(reader.close().ok()); } - ASSERT_TRUE(reader.close().ok()); } } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index 95a5aa4c935ec5..fdd540a0bd373a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -493,9 +493,18 @@ static Object parsePartitionValueFromString(String valueStr, Type icebergType, Z return (int) LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).toEpochDay(); case TIMESTAMP_NANO: // Match the sink cast: a date-only static partition denotes midnight. - LocalDateTime timestampNs = valueStr.length() == 10 - ? LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).atStartOfDay() - : LocalDateTime.parse(valueStr.replace('T', ' '), TIMESTAMP_PARTITION_FORMAT); + if (valueStr.length() == 10) { + return toEpochNanos(LocalDate.parse(valueStr, DateTimeFormatter.ISO_LOCAL_DATE).atStartOfDay()); + } + java.time.temporal.TemporalAccessor parsedNs = + TIMESTAMP_PARTITION_FORMAT.parse(valueStr.replace('T', ' ')); + LocalDateTime timestampNs = LocalDateTime.from(parsedNs); + ZoneOffset offset = parsedNs.query(java.time.temporal.TemporalQueries.offset()); + if (offset != null) { + // The sink casts offset literals to session-local civil time before writing NTZ. + // Overwrite filters must target that same partition, not discard the input offset. + timestampNs = LocalDateTime.ofInstant(timestampNs.toInstant(offset), zone); + } return toEpochNanos(timestampNs); case TIMESTAMP: return parseTimestampToMicros(valueStr, (TimestampType) icebergType, zone); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java index 6c6ae32f18138f..2c059d64cd0b7e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java @@ -56,6 +56,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; /** @@ -106,7 +107,7 @@ public class IcebergPredicateConverter { private static final List SIGNED_ZEROS = Collections.unmodifiableList(Arrays.asList(-0.0d, 0.0d)); private final Schema schema; - private final Set promotedDateFields = new LinkedHashSet<>(); + private final Set unsafeNanosFields = new LinkedHashSet<>(); private final ZoneId sessionZone; // The conversion matrix. SCAN = scan-time pushdown (BE re-filters, so widening is safe). CONFLICT = O5-2 // write-time optimistic conflict detection (no-missed-conflict, so widening is also safe) -- a port of @@ -135,14 +136,24 @@ public IcebergPredicateConverter(Schema schema, ZoneId sessionZone, Mode mode) { public static IcebergPredicateConverter forTable(Table table, Schema schema, ZoneId zone, Mode mode) { IcebergPredicateConverter converter = new IcebergPredicateConverter(schema, zone, mode); + Map history = table.schemas(); + // Iceberg allocates schema IDs from zero. Gaps mean cleanup removed type history, not + // necessarily its data files; without that history any nanos field may have DATE bounds. + int maxSchemaId = history.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1); + boolean incompleteHistory = history.isEmpty() + || history.size() != (long) maxSchemaId + 1; for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { if (field.type().typeId() != TypeID.TIMESTAMP_NANO) { continue; } - for (Schema historical : table.schemas().values()) { + if (incompleteHistory) { + converter.unsafeNanosFields.add(field.fieldId()); + continue; + } + for (Schema historical : history.values()) { Type oldType = historical.findType(field.fieldId()); if (oldType != null && oldType.typeId() == TypeID.DATE) { - converter.promotedDateFields.add(field.fieldId()); + converter.unsafeNanosFields.add(field.fieldId()); break; } } @@ -499,7 +510,7 @@ private Types.NestedField getPushdownField(String colName) { Types.NestedField field = schema.caseInsensitiveFindField(colName); // Iceberg 1.11 widens retained four-byte DATE bounds to unscaled longs. Pruning those // epoch-day bounds with nanosecond predicates can discard matching files in every planner. - return field != null && promotedDateFields.contains(field.fieldId()) ? null : field; + return field != null && unsafeNanosFields.contains(field.fieldId()) ? null : field; } /** diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 739eb1aecb5365..9c504ed00746f6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -915,6 +915,33 @@ public void dateOnlyNanosecondStaticOverwriteKeepsAdjacentPartition() { Assertions.assertTrue(remaining.stream().anyMatch(file -> file.path().toString().endsWith("replacement.parquet"))); } + @Test + public void offsetNanosecondStaticOverwriteKeepsAdjacentPartition() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("ts").build(); + Table table = catalog.createTable(id, schema, spec, props("format-version", "3")); + long midnight = 1704067200000000000L; + for (long nanos : new long[] {midnight, midnight - 8 * 3600_000_000_000L}) { + PartitionData partition = new PartitionData(spec.partitionType()); + partition.set(0, nanos); + table.newAppend().appendFile(DataFiles.builder(spec) + .withPath("s3://bucket/" + nanos + ".parquet").withFileSizeInBytes(1024) + .withRecordCount(1).withPartition(partition).withFormat(FileFormat.PARQUET).build()).commit(); + } + IcebergConnectorTransaction txn = txnFor(opsReturning(table), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", + overwriteStaticCtx(table, Collections.singletonMap("ts", "2024-01-01 00:00:00+08:00"))); + txn.addCommitData(commitBytes(dataFileItem("s3://bucket/replacement.parquet", 1L, 1024L, + Collections.singletonList("2023-12-31 16:00:00.000000000")))); + txn.commit(); + List remaining = currentDataFiles(catalog.loadTable(id)); + Assertions.assertEquals(2, remaining.size()); + Assertions.assertTrue(remaining.stream().anyMatch(file -> file.partition().get(0, Long.class) == midnight)); + Assertions.assertTrue(remaining.stream().anyMatch(file -> file.path().toString().endsWith("replacement.parquet"))); + } + @Test public void overwriteStaticPartitionRejectsUnmatchedPartitionField() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 9344a9d961e0e7..a42e4ec1766c3f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -4076,13 +4076,27 @@ public void promotedDateBoundsDoNotPruneNanosecondMatches() { new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-02 00:00:00"))).size(), "fields without historical DATE bounds must retain pushdown"); } + // Metadata cleanup may drop the writing schema while the retained snapshot still uses its file. + table.newAppend().appendFile(DataFiles.builder(table.spec()) + .withPath("/data/new-nanos.parquet").withFileSizeInBytes(100).withRecordCount(1) + .withFormat(FileFormat.PARQUET).build()).commit(); + table.expireSnapshots().expireOlderThan(System.currentTimeMillis() + 1000) + .retainLast(1).cleanExpiredMetadata(true).commit(); + table.refresh(); + Assertions.assertTrue(table.schemas().values().stream() + .noneMatch(retained -> Types.DateType.get().equals(retained.findType(1)))); + for (IcebergPredicateConverter.Mode mode : IcebergPredicateConverter.Mode.values()) { + Assertions.assertTrue(IcebergPredicateConverter.forTable( + table, table.schema(), java.time.ZoneOffset.UTC, mode).convert(filter).isEmpty(), + "schema cleanup must not re-enable unsafe DATE bound pruning in " + mode); + } ConnectorScanRequest request = ConnectorScanRequest.builder( new IcebergTableHandle("db1", "promoted_date"), Collections.emptyList()) .filter(Optional.of(filter)).build(); List sdk = new IcebergScanPlanProvider( IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .planScan(emptySession(), request); - Assertions.assertEquals(1, sdk.size(), "SDK planning must preserve the matching DATE file"); + Assertions.assertEquals(2, sdk.size(), "SDK planning must preserve the matching DATE file after cleanup"); IcebergManifestCache cache = new IcebergManifestCache(); List cached = manifestProvider(manifestCacheProps(), table, cache) .planScan(emptySession(), request); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java index 3d87eebf7a0853..2a141c13659e0e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java @@ -97,6 +97,21 @@ public void predicatesPreserveNanosecondsInEveryMode() { } } + @Test + public void offsetPartitionsUseSessionCivilTimeAndKeepNanoseconds() { + String input = "2024-01-01 00:00:00.123456789+08:00"; + String[] zones = {"UTC", "Asia/Shanghai", "America/New_York"}; + String[] expected = {"2023-12-31 16:00:00.123456789", "2024-01-01 00:00:00.123456789", + "2023-12-31 11:00:00.123456789"}; + for (int i = 0; i < zones.length; i++) { + ZoneId zone = ZoneId.of(zones[i]); + Assertions.assertEquals(IcebergPartitionUtils.parsePartitionValueFromString( + expected[i], Types.TimestampNanoType.withoutZone(), zone), + IcebergPartitionUtils.parsePartitionValueFromString( + input, Types.TimestampNanoType.withoutZone(), zone)); + } + } + @Test public void writeDefaultsPreserveNanoseconds() { for (long nanos : new long[] {Long.MIN_VALUE, -1, 0, 1, 123456789, Long.MAX_VALUE}) { diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out index 6e170b9882bf71..47e0eeffc34e23 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -87,6 +87,17 @@ entries map Yes true \N -- !parquet_identity_static_date -- 2024-01-01 00:00:00.000000000 +-- !parquet_identity_static_offset -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 +8 2024-01-01 00:00:00.000000000 +10 2023-12-31 16:00:00.000000000 + -- !parquet_year -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 @@ -260,6 +271,17 @@ entries map Yes true \N -- !orc_identity_static_date -- 2024-01-01 00:00:00.000000000 +-- !orc_identity_static_offset -- +1 2024-01-02 03:04:05.123456789 +2 2024-01-02 03:04:05.123456788 +3 1969-12-31 23:59:58.999999999 +4 1970-01-01 00:00:00.000000001 +5 \N +6 1677-09-21 00:12:43.145224192 +7 2262-04-11 23:47:16.854775807 +8 2024-01-01 00:00:00.000000000 +10 2023-12-31 16:00:00.000000000 + -- !orc_year -- 1 2024-01-02 03:04:05.123456789 2 2024-01-02 03:04:05.123456788 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy index 6a06d496e46597..9da757c7d13980 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -108,6 +108,13 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { // Static partition commit parsing must accept the same date-only input as the sink cast. sql "INSERT OVERWRITE TABLE ${partitionTable} PARTITION(ts='2024-01-01') SELECT 8" "qt_${format}_identity_static_date" "SELECT CAST(ts AS STRING) FROM ${partitionTable} WHERE id=8" + // An offset literal must replace the sink's normalized partition, preserving midnight. + sql "SET time_zone='UTC'" + sql "INSERT INTO ${partitionTable} VALUES (9, '2023-12-31 16:00:00')" + sql """INSERT OVERWRITE TABLE ${partitionTable} + PARTITION(ts='2024-01-01 00:00:00+08:00') SELECT 10""" + "qt_${format}_identity_static_offset" "SELECT * FROM ${partitionTable} ORDER BY id" + sql "SET time_zone='Asia/Shanghai'" } } } From 7ec459959f83345c3398d73e5c7687201f7c042a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 11:34:28 +0800 Subject: [PATCH 09/13] [refactor](be) Isolate external schema evolution casts ### What problem does this PR solve? Related PR: #68934 Problem Summary: Keep external file-to-table promotion policy out of ordinary Cast. Introduce EvolutionCast to enforce DATE-to-nanosecond overflow errors through the existing checked conversion, without changing execution-layer functions or ordinary SQL cast semantics. Use the expression for projection, localized predicates, table materialization, and equality-delete keys and defaults. ### Release note None ### Check List (For Author) - Test: 384 BE unit tests, including seven expression tests and retained Parquet DATE schema evolution; ASAN BE build; clang-format 16; build hygiene; clang-tidy on changed lines. - Behavior changed: No; preserve strict evolution validation and ordinary SQL cast behavior. - Does this need documentation: No --- be/src/format_v2/column_mapper.cpp | 5 +- be/src/format_v2/expr/cast.cpp | 8 - be/src/format_v2/expr/cast.h | 8 +- be/src/format_v2/expr/evolution_cast.cpp | 41 ++++ be/src/format_v2/expr/evolution_cast.h | 40 ++++ be/src/format_v2/table/iceberg_reader.cpp | 10 +- be/src/format_v2/table_reader.h | 4 +- .../format_v2/expr/evolution_cast_test.cpp | 182 ++++++++++++++++++ 8 files changed, 276 insertions(+), 22 deletions(-) create mode 100644 be/src/format_v2/expr/evolution_cast.cpp create mode 100644 be/src/format_v2/expr/evolution_cast.h create mode 100644 be/test/format_v2/expr/evolution_cast_test.cpp diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 02b08f450e3f4e..37db579271676e 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -51,6 +51,7 @@ #include "exprs/vslot_ref.h" #include "format_v2/column_mapper_nested.h" #include "format_v2/expr/cast.h" +#include "format_v2/expr/evolution_cast.h" #include "format_v2/file_reader.h" #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" @@ -1104,7 +1105,7 @@ static VExprSPtr cast_file_expr_to_table_type(const VExprSPtr& file_expr, DORIS_CHECK(rewrite_context != nullptr); // Localized filters perform the same evolution as projection; SQL non-strict casts must // not replace an overflowing retained DATE with NULL and filter it out before materialization. - auto cast_expr = Cast::create_shared(table_type, true); + auto cast_expr = EvolutionCast::create_shared(table_type); cast_expr->add_child(file_expr); rewrite_context->add_created_expr(cast_expr); return cast_expr; @@ -2191,7 +2192,7 @@ static void rebuild_projection(ColumnMapping* mapping, LocalIndex block_position return; } - auto expr = Cast::create_shared(mapping->table_type, true); + auto expr = EvolutionCast::create_shared(mapping->table_type); expr->add_child(VSlotRef::create_shared(cast_set(block_position.value()), cast_set(block_position.value()), -1, mapping->file_type, mapping->file_column_name)); diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index 8c529f18b2fa3b..c1435732a9e136 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -27,7 +27,6 @@ #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/block/columns_with_type_and_name.h" -#include "core/data_type/data_type_nullable.h" #include "exprs/function/simple_function_factory.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" @@ -66,13 +65,6 @@ Status Cast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContex return Status::InternalError("Could not find function {} ", _expr_name); } VExpr::register_function_context(state, context); - if (_is_schema_evolution && - remove_nullable(_children[0]->data_type())->get_primitive_type() == TYPE_DATEV2 && - remove_nullable(_data_type)->get_primitive_type() == TYPE_TIMESTAMP_NS) { - // A retained DATE file must fail outside the promoted nanosecond range, never synthesize NULL. - // Keep this independent of SQL CAST semantics and the session's strict-cast setting. - context->fn_context(_fn_context_index)->set_enable_strict_mode(true); - } _prepare_finished = true; return Status::OK(); } diff --git a/be/src/format_v2/expr/cast.h b/be/src/format_v2/expr/cast.h index 07f54bb29d7d28..85223a046d744b 100644 --- a/be/src/format_v2/expr/cast.h +++ b/be/src/format_v2/expr/cast.h @@ -34,12 +34,11 @@ class VExprContext; namespace doris::format { -class Cast final : public VExpr { +class Cast : public VExpr { ENABLE_FACTORY_CREATOR(Cast); public: - Cast(const DataTypePtr& type, bool is_schema_evolution = false) - : _is_schema_evolution(is_schema_evolution) { + Cast(const DataTypePtr& type) { _node_type = TExprNodeType::CAST_EXPR; _opcode = TExprOpcode::CAST; _data_type = type; @@ -59,12 +58,11 @@ class Cast final : public VExpr { bool is_safe_to_execute_on_selected_rows() const override { return false; } Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); - *cloned_expr = Cast::create_shared(_data_type, _is_schema_evolution); + *cloned_expr = Cast::create_shared(_data_type); return Status::OK(); } private: - bool _is_schema_evolution; Status _do_execute(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const; std::string _expr_name; diff --git a/be/src/format_v2/expr/evolution_cast.cpp b/be/src/format_v2/expr/evolution_cast.cpp new file mode 100644 index 00000000000000..70d878474f7c31 --- /dev/null +++ b/be/src/format_v2/expr/evolution_cast.cpp @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/evolution_cast.h" + +#include "core/data_type/data_type_nullable.h" +#include "exprs/vexpr_context.h" + +namespace doris::format { + +Status EvolutionCast::prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) { + RETURN_IF_ERROR(Cast::prepare(state, desc, context)); + if (remove_nullable(_children[0]->data_type())->get_primitive_type() == TYPE_DATEV2 && + remove_nullable(_data_type)->get_primitive_type() == TYPE_TIMESTAMP_NS) { + // Iceberg DATE promotion must fail outside the nanosecond range, even in non-strict SQL + // sessions. Otherwise a localized IS NOT NULL filter can silently discard retained rows. + context->fn_context(_fn_context_index)->set_enable_strict_mode(true); + } + return Status::OK(); +} + +std::string EvolutionCast::debug_string() const { + return "Evolution" + Cast::debug_string(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/expr/evolution_cast.h b/be/src/format_v2/expr/evolution_cast.h new file mode 100644 index 00000000000000..25b5c02cc70202 --- /dev/null +++ b/be/src/format_v2/expr/evolution_cast.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "format_v2/expr/cast.h" + +namespace doris::format { + +// File-to-table conversions own external schema evolution validation; SQL casts do not. +class EvolutionCast final : public Cast { + ENABLE_FACTORY_CREATOR(EvolutionCast); + +public: + explicit EvolutionCast(const DataTypePtr& type) : Cast(type) {} + + Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; + std::string debug_string() const override; + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = EvolutionCast::create_shared(_data_type); + return Status::OK(); + } +}; + +} // namespace doris::format diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b33265259c4a5b..1751c8d1f87d62 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -48,8 +48,8 @@ #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_default_value.h" -#include "format_v2/expr/cast.h" #include "format_v2/expr/equality_delete_predicate.h" +#include "format_v2/expr/evolution_cast.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" #include "format_v2/parquet/reader/column_reader.h" @@ -653,7 +653,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit *key_expr = std::move(literal); return Status::OK(); } - auto cast_expr = Cast::create_shared(delete_key_type, true); + auto cast_expr = EvolutionCast::create_shared(delete_key_type); cast_expr->add_child(literal); *key_expr = std::move(cast_expr); return Status::OK(); @@ -1511,7 +1511,7 @@ Status IcebergTableReader::_build_missing_equality_delete_key_expr( DORIS_CHECK(table_leaf != nullptr); DORIS_CHECK(table_leaf->type != nullptr); if (!table_leaf->type->equals(*filter.key_types[key_idx])) { - auto cast_expr = Cast::create_shared(filter.key_types[key_idx], true); + auto cast_expr = EvolutionCast::create_shared(filter.key_types[key_idx]); cast_expr->add_child(default_expr); default_expr = std::move(cast_expr); } @@ -1565,7 +1565,7 @@ Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe if (field->type->equals(*filter.key_types[idx])) { delete_predicate->add_child(key_expr); } else { - auto cast_expr = Cast::create_shared(filter.key_types[idx], true); + auto cast_expr = EvolutionCast::create_shared(filter.key_types[idx]); cast_expr->add_child(key_expr); delete_predicate->add_child(cast_expr); } @@ -1834,7 +1834,7 @@ Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDe if (!key_expr->data_type()->equals(*key_type)) { // Historical delete values share the promoted comparison domain. DATE-to-nanos // must report overflow rather than turn a non-NULL equality key into NULL. - auto cast_expr = Cast::create_shared(key_type, true); + auto cast_expr = EvolutionCast::create_shared(key_type); cast_expr->add_child(key_expr); key_expr = std::move(cast_expr); } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index f8b1d529a473e8..0a721ae002fe85 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -57,8 +57,8 @@ #include "format/table/deletion_vector.h" #include "format_v2/column_data.h" #include "format_v2/column_mapper.h" -#include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" +#include "format_v2/expr/evolution_cast.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/schema_projection.h" @@ -1357,7 +1357,7 @@ class TableReader { Block cast_block; cast_block.insert({*column, input_type, column_name}); auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name); - auto cast_expr = Cast::create_shared(cast_type, true); + auto cast_expr = EvolutionCast::create_shared(cast_type); cast_expr->add_child(std::move(slot_ref)); auto cast_ctx = VExprContext::create_shared(std::move(cast_expr)); RowDescriptor row_desc; diff --git a/be/test/format_v2/expr/evolution_cast_test.cpp b/be/test/format_v2/expr/evolution_cast_test.cpp new file mode 100644 index 00000000000000..e9cd32d656298f --- /dev/null +++ b/be/test/format_v2/expr/evolution_cast_test.cpp @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/expr/evolution_cast.h" + +#include + +#include "core/block/block.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris::format { + +class EvolutionCastTest : public testing::Test { +protected: + static DateV2Value date(int year, int month, int day) { + DateV2Value value; + value.unchecked_set_time(year, month, day, 0, 0, 0, 0); + return value; + } + + Status execute(const VExprSPtr& expr, Block* block, int* result) { + auto context = VExprContext::create_shared(expr); + auto status = context->prepare(&state, RowDescriptor()); + if (status.ok()) { + status = context->open(&state); + } + if (status.ok()) { + status = context->execute(block, result); + } + context->close(); + return status; + } + + DataTypePtr source_type = make_nullable(std::make_shared()); + DataTypePtr target_type = make_nullable(std::make_shared()); + MockRuntimeState state; +}; + +TEST_F(EvolutionCastTest, OverflowIsIndependentOfSessionAndSurvivesClone) { + for (bool strict : {false, true}) { + state.set_enable_strict_cast(strict); + for (auto value : + {date(1000, 1, 1), date(1677, 9, 21), date(2262, 4, 12), date(9999, 12, 31)}) { + for (bool clone : {false, true}) { + SCOPED_TRACE(testing::Message() << "strict=" << strict << " clone=" << clone); + VExprSPtr expr = EvolutionCast::create_shared(target_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, source_type, "source")); + if (clone) { + VExprSPtr cloned; + ASSERT_TRUE(expr->deep_clone(&cloned).ok()); + expr = std::move(cloned); + } + Block block; + block.insert({ColumnHelper::create_nullable_column({value}, {0}), + source_type, "source"}); + int result = -1; + auto status = execute(expr, &block, &result); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), std::string::npos) + << status; + } + } + } +} + +TEST_F(EvolutionCastTest, ValidDatesAndNullRemainUnchanged) { + state.set_enable_strict_cast(false); + auto expr = EvolutionCast::create_shared(target_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, source_type, "source")); + Block block; + // A NULL payload may be outside the target range and must not trigger overflow. + block.insert({ColumnHelper::create_nullable_column( + {date(1677, 9, 22), date(2262, 4, 11), date(1000, 1, 1)}, {0, 0, 1}), + source_type, "source"}); + int result = -1; + ASSERT_TRUE(execute(expr, &block, &result).ok()); + const auto& column = *block.get_by_position(result).column; + EXPECT_EQ(target_type->to_string(column, 0), "1677-09-22 00:00:00.000000000"); + EXPECT_EQ(target_type->to_string(column, 1), "2262-04-11 00:00:00.000000000"); + EXPECT_TRUE(column.is_null_at(2)); +} + +TEST_F(EvolutionCastTest, ConstantOverflowFailsDuringOpen) { + state.set_enable_strict_cast(false); + auto expr = EvolutionCast::create_shared(target_type); + expr->add_child(VLiteral::create_shared(std::make_shared(), + Field::create_field(date(1000, 1, 1)))); + auto context = VExprContext::create_shared(expr); + ASSERT_TRUE(context->prepare(&state, RowDescriptor()).ok()); + auto status = context->open(&state); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), std::string::npos) << status; + context->close(); +} + +TEST_F(EvolutionCastTest, OrdinaryCastPreservesSessionBehavior) { + for (bool strict : {false, true}) { + state.set_enable_strict_cast(strict); + auto expr = Cast::create_shared(target_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, source_type, "source")); + Block block; + block.insert({ColumnHelper::create_nullable_column({date(1000, 1, 1)}, {0}), + source_type, "source"}); + int result = -1; + auto status = execute(expr, &block, &result); + if (strict) { + EXPECT_FALSE(status.ok()); + } else { + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(block.get_by_position(result).column->is_null_at(0)); + } + } +} + +TEST_F(EvolutionCastTest, RuntimeContextCloneRetainsOverflowCheck) { + state.set_enable_strict_cast(false); + auto expr = EvolutionCast::create_shared(target_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, source_type, "source")); + auto context = VExprContext::create_shared(expr); + ASSERT_TRUE(context->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(context->open(&state).ok()); + VExprContextSPtr cloned; + ASSERT_TRUE(context->clone(&state, cloned).ok()); + Block block; + block.insert({ColumnHelper::create_nullable_column({date(1000, 1, 1)}, {0}), + source_type, "source"}); + int result = -1; + auto status = cloned->execute(&block, &result); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("TIMESTAMP_NS overflow"), std::string::npos) << status; + cloned->close(); + context->close(); +} + +TEST_F(EvolutionCastTest, EmptyInputRemainsEmpty) { + state.set_enable_strict_cast(false); + auto expr = EvolutionCast::create_shared(target_type); + expr->add_child(VSlotRef::create_shared(0, 0, -1, source_type, "source")); + Block block; + block.insert( + {ColumnHelper::create_nullable_column({}, {}), source_type, "source"}); + int result = -1; + ASSERT_TRUE(execute(expr, &block, &result).ok()); + EXPECT_EQ(block.get_by_position(result).column->size(), 0); +} + +TEST_F(EvolutionCastTest, NumericPromotionUsesExistingConversion) { + auto expr = EvolutionCast::create_shared(std::make_shared()); + expr->add_child(VSlotRef::create_shared(0, 0, -1, std::make_shared(), "source")); + Block block; + block.insert(ColumnHelper::create_column_with_name({1, -2, 3})); + int result = -1; + ASSERT_TRUE(execute(expr, &block, &result).ok()); + const auto& column = assert_cast(*block.get_by_position(result).column); + EXPECT_EQ(column.get_data()[0], 1); + EXPECT_EQ(column.get_data()[1], -2); + EXPECT_EQ(column.get_data()[2], 3); +} + +} // namespace doris::format From 19b0a213f6ac6e2bc31c281352e23ae2dee3bfeb Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 12:23:34 +0800 Subject: [PATCH 10/13] [refactor](be) Centralize ORC timestamp decoding context ### What problem does this PR solve? Related PR: #68934 Problem Summary: ORC timestamp decoding scattered reader strategy flags and optional timezone overrides across scan setup and SerDes. Centralize the existing SDK workaround policy in OrcTimestampReadPlan and pass an explicit reader/session timezone context to SerDes. Select the decoding timezone by ORC physical timestamp kind and preserve nested view propagation and stripe transitions without modifying the SDK. ### Release note None ### Check List (For Author) - Test: 827 focused BE unit tests and the full ASAN BE build passed; clang-format 16, clang-tidy on modified lines, and BE header hygiene passed. - Behavior changed: No - Does this need documentation: No --- .../data_type_datetimev2_serde.cpp | 10 +- be/src/core/data_type_serde/data_type_serde.h | 6 +- .../data_type_timestamp_ns_serde.cpp | 8 +- .../core/data_type_serde/orc_serde_utils.cpp | 5 +- .../orc_timestamp_decode_context.cpp | 34 +++++ .../orc_timestamp_decode_context.h | 39 ++++++ be/src/format_v2/orc/orc_reader.cpp | 126 +++++++++++------- .../data_type/data_type_timestamp_ns_test.cpp | 4 +- .../orc_timestamp_decode_context_test.cpp | 101 ++++++++++++++ be/test/format_v2/orc/orc_reader_test.cpp | 4 +- 10 files changed, 276 insertions(+), 61 deletions(-) create mode 100644 be/src/core/data_type_serde/orc_timestamp_decode_context.cpp create mode 100644 be/src/core/data_type_serde/orc_timestamp_decode_context.h create mode 100644 be/test/core/data_type_serde/orc_timestamp_decode_context_test.cpp diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index acac0d6e385fe6..786833599c39a8 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -32,6 +32,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" @@ -1046,15 +1047,12 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, DORIS_CHECK(view.batch != nullptr); const auto kind = view.file_type->getKind(); DORIS_CHECK(kind == ::orc::TypeKind::TIMESTAMP || kind == ::orc::TypeKind::TIMESTAMP_INSTANT); - DORIS_CHECK(view.timezone != nullptr); + DORIS_CHECK(view.timestamp_context != nullptr); if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { return Status::OK(); } - const auto* timezone = view.file_type->getKind() == ::orc::TIMESTAMP && - view.plain_timestamp_timezone != nullptr - ? view.plain_timestamp_timezone - : view.timezone; - return decode_timestamp_orc_values(column, view, *timezone); + return decode_timestamp_orc_values(column, view, + view.timestamp_context->timezone_for(*view.file_type)); } } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index b7df1b019bc5c3..2a57adaf5d3aa6 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -118,16 +118,16 @@ struct FieldInfo { int precision = 0; }; +struct OrcTimestampDecodeContext; + struct OrcDecodedColumnView { const orc::Type* file_type = nullptr; const orc::Type* selected_type = nullptr; const orc::ColumnVectorBatch* batch = nullptr; size_t rows = 0; const std::vector* selected_rows = nullptr; - const cctz::time_zone* timezone = nullptr; + const OrcTimestampDecodeContext* timestamp_context = nullptr; bool enable_mapping_timestamp_tz = false; - // Plain timestamps may retain the writer's carrier; instant timestamps still use the session zone. - const cctz::time_zone* plain_timestamp_timezone = nullptr; }; // Deserialize means read from different file format or memory format, diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 7fd8e589eb809f..5fd2c968fc11c5 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -30,6 +30,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" #include "core/data_type_serde/parquet_timestamp.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_timestamp_ns.h" @@ -477,10 +478,9 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows); - const auto* source_timezone = view.plain_timestamp_timezone == nullptr - ? view.timezone - : view.plain_timestamp_timezone; - const auto timezone = source_timezone == nullptr ? cctz::utc_time_zone() : *source_timezone; + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.timestamp_context != nullptr); + const auto& timezone = view.timestamp_context->timezone_for(*view.file_type); for (size_t row = 0; row < rows; ++row) { const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows); if (orc_serde_utils::orc_row_is_null(batch, source)) { diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp b/be/src/core/data_type_serde/orc_serde_utils.cpp index e4d78c2b7a2901..f73f708b6297df 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.cpp +++ b/be/src/core/data_type_serde/orc_serde_utils.cpp @@ -20,6 +20,7 @@ #include "common/cast_set.h" #include "common/check.h" #include "core/column/column_array.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" namespace doris::orc_serde_utils { @@ -106,7 +107,9 @@ DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view, DecodedColumnView view; view.value_kind = value_kind; view.row_count = cast_set(orc_decode_row_count(orc_view.rows, orc_view.selected_rows)); - view.timezone = orc_view.timezone; + if (orc_view.timestamp_context != nullptr) { + view.timezone = &orc_view.timestamp_context->session_timezone; + } return view; } diff --git a/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp b/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp new file mode 100644 index 00000000000000..53152b57670ecb --- /dev/null +++ b/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/orc_timestamp_decode_context.h" + +#include + +#include "common/check.h" + +namespace doris { + +const cctz::time_zone& OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type) const { + const auto kind = file_type.getKind(); + DORIS_CHECK(kind == orc::TIMESTAMP || kind == orc::TIMESTAMP_INSTANT); + // Plain timestamps reconstruct the SDK reader's civil value. Instant-to-DATETIME conversion + // uses the session zone, which must not follow a plain sibling's stripe writer timezone. + return kind == orc::TIMESTAMP ? reader_timezone : session_timezone; +} + +} // namespace doris diff --git a/be/src/core/data_type_serde/orc_timestamp_decode_context.h b/be/src/core/data_type_serde/orc_timestamp_decode_context.h new file mode 100644 index 00000000000000..228c781d0ba2f7 --- /dev/null +++ b/be/src/core/data_type_serde/orc_timestamp_decode_context.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace orc { +class Type; +} + +namespace doris { + +// Reader and session zones have different roles even when a batch mixes both timestamp kinds. +struct OrcTimestampDecodeContext { + explicit OrcTimestampDecodeContext(const cctz::time_zone& timezone) + : reader_timezone(timezone), session_timezone(timezone) {} + + const cctz::time_zone& timezone_for(const orc::Type& file_type) const; + + cctz::time_zone reader_timezone; + cctz::time_zone session_timezone; +}; + +} // namespace doris diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 4381e7d37b5730..b6f374502fd8bb 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -68,6 +68,7 @@ #include "core/data_type/data_type_varbinary.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/orc_serde_utils.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" #include "core/types.h" #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" @@ -195,19 +196,6 @@ Status set_orc_reader_timezone(const std::string& timezone, return Status::OK(); } -bool has_timestamp(const ::orc::Type& type, bool include_instant) { - if (type.getKind() == ::orc::TIMESTAMP || - (include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) { - return true; - } - for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { - if (has_timestamp(*type.getSubtype(child), include_instant)) { - return true; - } - } - return false; -} - // selected_rows is a source-row remap produced by ORC lazy callback: // predicate columns are decoded first, then surviving row ids drive follower decodes. size_t decode_row_count(size_t rows, const std::vector* selected_rows) { @@ -345,6 +333,72 @@ bool is_virtual_column(format::LocalColumnId file_column_id) { return is_row_position_column(file_column_id) || is_global_rowid_column(file_column_id); } +// Keep SDK workarounds and their scan requirements in one plan. SerDes receive only the +// resulting decode context and never need to infer which reader strategy produced a batch. +class OrcTimestampReadPlan { +public: + void configure(const ::orc::Type& file_schema, + const std::vector& read_columns, + const cctz::time_zone& session_timezone) { + _context = OrcTimestampDecodeContext(session_timezone); + // An unprojected NANOS sibling can affect the SDK's ordinary timestamp decoding. + const bool needs_writer_timezone = + orc_serde_utils::has_nanosecond_timestamp(file_schema) && + std::ranges::any_of(read_columns, [&](const auto column_id) { + return !is_virtual_column(column_id) && + has_timestamp(*file_schema.getSubtype(column_id.value()), false); + }); + _timezone_source = + needs_writer_timezone ? TimezoneSource::STRIPE_WRITER : TimezoneSource::SESSION; + } + + bool requires_stripe_reader() const { + return _timezone_source == TimezoneSource::STRIPE_WRITER; + } + + bool requires_dense_decoding(const ::orc::Type& column_type) const { + // The pinned SDK's selected timestamp decoder checks a fraction at the wrong row index. + // Decode timestamp leaders densely while retaining lazy reads for other followers. + return requires_stripe_reader() && has_timestamp(column_type, true); + } + + Status configure_stripe(const ::orc::StripeInformation& stripe, + ::orc::RowReaderOptions* options) { + DORIS_CHECK(requires_stripe_reader()); + const auto& name = stripe.getWriterTimezone(); + const auto writer_timezone = name.empty() ? "GMT" : name; + if (!TimezoneUtils::find_cctz_time_zone(writer_timezone, _context.reader_timezone)) { + return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", + writer_timezone); + } + // Matching zones preserves the SDK's UTC carrier until its negative-fraction correction + // is complete. SerDes then apply historical offsets to that original second with cctz. + options->setTimezoneName(writer_timezone); + return Status::OK(); + } + + const OrcTimestampDecodeContext& decode_context() const { return _context; } + +private: + enum class TimezoneSource { SESSION, STRIPE_WRITER }; + + static bool has_timestamp(const ::orc::Type& type, bool include_instant) { + if (type.getKind() == ::orc::TIMESTAMP || + (include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) { + return true; + } + for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { + if (has_timestamp(*type.getSubtype(child), include_instant)) { + return true; + } + } + return false; + } + + TimezoneSource _timezone_source = TimezoneSource::SESSION; + OrcTimestampDecodeContext _context {cctz::utc_time_zone()}; +}; + format::ColumnDefinition nullable_global_rowid_column_definition() { auto field = format::global_rowid_column_definition(); field.type = make_nullable(field.type); @@ -778,8 +832,7 @@ struct OrcReaderScanState { ::orc::RowReaderOptions row_reader_options; // projection + filter + SARG + stripe range std::string timezone = TimezoneUtils::default_time_zone; cctz::time_zone timezone_obj; - bool match_writer_timezone = false; - cctz::time_zone writer_timezone_obj; + OrcTimestampReadPlan timestamp_plan; std::unique_ptr<::orc::RowReader> row_reader; const ::orc::Type* selected_type = nullptr; std::unique_ptr<::orc::ColumnVectorBatch> batch; @@ -1024,7 +1077,7 @@ int64_t OrcReader::get_total_rows() const { if (_state->stripe_ranges_applied && _state->selected_stripe_ranges.empty()) { return 0; } - if (_state->match_writer_timezone && _state->stripe_ranges_applied) { + if (_state->timestamp_plan.requires_stripe_reader() && _state->stripe_ranges_applied) { uint64_t rows = 0; for (const auto& range : _state->selected_stripe_ranges) { rows += range.end_row - range.first_row; @@ -1356,7 +1409,7 @@ Status OrcReader::open(std::shared_ptr request) { for (const int64_t row_id : *_request->row_ids) { uint64_t first_row = _state->row_reader_range_first_row; uint64_t end_row = _state->row_reader_range_end_row; - if (_state->match_writer_timezone) { + if (_state->timestamp_plan.requires_stripe_reader()) { const auto& ranges = _state->selected_stripe_ranges; while (range_index + 1 < ranges.size() && std::cmp_greater_equal(row_id, ranges[range_index].end_row)) { @@ -1426,21 +1479,14 @@ Status OrcReader::_configure_row_reader_projection() { DORIS_CHECK(projection != nullptr); has_complex_projection = has_complex_projection || has_pruned_projection(*projection); } - // A hidden NANOS sibling can affect ordinary timestamps, but scans without civil - // timestamp columns do not need stripe-specific reader timezones. - _state->match_writer_timezone = - orc_serde_utils::has_nanosecond_timestamp(*_state->root_type) && - std::ranges::any_of(_state->read_columns, [&](const auto column_id) { - return !is_virtual_column(column_id) && - has_timestamp(*_state->root_type->getSubtype(column_id.value()), false); - }); + _state->timestamp_plan.configure(*_state->root_type, _state->read_columns, + _state->timezone_obj); auto filter_projections = _request->predicate_columns; - if (_state->match_writer_timezone && _state->orc_lazy_read_enabled) { - // The pinned SDK's selected timestamp decoder tests a fraction at the wrong row index. - // Decode timestamp leaders densely; keep lazy reads for all other follower columns. + if (_state->orc_lazy_read_enabled) { for (const auto column_id : _state->read_columns) { if (!is_virtual_column(column_id) && - has_timestamp(*_state->root_type->getSubtype(column_id.value()), true) && + _state->timestamp_plan.requires_dense_decoding( + *_state->root_type->getSubtype(column_id.value())) && std::ranges::none_of(filter_projections, [column_id](const auto& projection) { return projection.column_id() == column_id; })) { @@ -1716,7 +1762,7 @@ Status OrcReader::_select_stripe_ranges_by_statistics() { } Status OrcReader::_prepare_timestamp_stripe_ranges() { - if (!_state->match_writer_timezone) { + if (!_state->timestamp_plan.requires_stripe_reader()) { return Status::OK(); } std::vector stripes; @@ -1769,18 +1815,11 @@ Status OrcReader::_apply_current_stripe_range() { DORIS_CHECK(_state->current_stripe_range < _state->selected_stripe_ranges.size()); const auto& stripe_range = _state->selected_stripe_ranges[_state->current_stripe_range]; _state->row_reader_options.range(stripe_range.offset, stripe_range.length); - if (_state->match_writer_timezone) { + if (_state->timestamp_plan.requires_stripe_reader()) { try { const auto stripe = _state->reader->getStripe(stripe_range.first_stripe); - const auto& name = stripe->getWriterTimezone(); - const auto writer_timezone = name.empty() ? "GMT" : name; - if (!TimezoneUtils::find_cctz_time_zone(writer_timezone, _state->writer_timezone_obj)) { - return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", - writer_timezone); - } - // Matching zones preserves the SDK's UTC carrier until its negative-fraction - // correction is complete. cctz then applies historical offsets to that original second. - _state->row_reader_options.setTimezoneName(writer_timezone); + RETURN_IF_ERROR( + _state->timestamp_plan.configure_stripe(*stripe, &_state->row_reader_options)); } catch (const std::exception& e) { if (is_orc_stop(_io_ctx.get(), e)) { return Status::EndOfFile("stop"); @@ -2088,11 +2127,8 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type view.batch = &batch; view.rows = rows; view.selected_rows = selected_rows; - view.timezone = &_state->timezone_obj; + view.timestamp_context = &_state->timestamp_plan.decode_context(); view.enable_mapping_timestamp_tz = _enable_mapping_timestamp_tz; - if (_state->match_writer_timezone) { - view.plain_timestamp_timezone = &_state->writer_timezone_obj; - } return column_type->get_serde()->read_column_from_orc(*column, view); } @@ -2131,7 +2167,7 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { } fetched_row_id = static_cast((*_request->row_ids)[_state->next_row_id]); // Row ids are sorted globally and can cross the per-writer-timezone ranges. - while (_state->match_writer_timezone && + while (_state->timestamp_plan.requires_stripe_reader() && *fetched_row_id >= _state->row_reader_range_end_row) { bool advanced = false; RETURN_IF_ERROR(_advance_to_next_stripe_range(&advanced)); diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp index 6496cb19af93d6..0551e9d94eaf8a 100644 --- a/be/test/core/data_type/data_type_timestamp_ns_test.cpp +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -36,6 +36,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/data_type_timestamp_ns_serde.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" #include "core/string_buffer.hpp" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_parameters.h" @@ -709,7 +710,8 @@ TEST(DataTypeTimeStampNsTest, OrcRoundTripPreservesNanoseconds) { view.file_type = orc_type.get(); view.batch = &batch; view.rows = 4; - view.timezone = &timezone; + const OrcTimestampDecodeContext timestamp_context(timezone); + view.timestamp_context = ×tamp_context; ASSERT_TRUE(serde->read_column_from_orc(*output, view).ok()); const auto& result = assert_cast(*output).get_data(); ASSERT_EQ(result.size(), values.size()); diff --git a/be/test/core/data_type_serde/orc_timestamp_decode_context_test.cpp b/be/test/core/data_type_serde/orc_timestamp_decode_context_test.cpp new file mode 100644 index 00000000000000..0aa72360c155db --- /dev/null +++ b/be/test/core/data_type_serde/orc_timestamp_decode_context_test.cpp @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/orc_timestamp_decode_context.h" + +#include + +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/orc_serde_utils.h" + +namespace doris { + +TEST(OrcTimestampDecodeContextTest, SessionIsTheDefaultReaderTimezone) { + const OrcTimestampDecodeContext context(cctz::fixed_time_zone(cctz::seconds(8 * 3600))); + const auto plain = orc::createPrimitiveType(orc::TIMESTAMP); + const auto instant = orc::createPrimitiveType(orc::TIMESTAMP_INSTANT); + EXPECT_EQ(context.timezone_for(*plain).name(), context.session_timezone.name()); + EXPECT_EQ(context.timezone_for(*instant).name(), context.session_timezone.name()); +} + +TEST(OrcTimestampDecodeContextTest, PhysicalTypeSelectsCivilOrInstantTimezone) { + OrcTimestampDecodeContext context(cctz::fixed_time_zone(cctz::seconds(8 * 3600))); + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &context.reader_timezone)); + orc::TimestampVectorBatch batch(1, *orc::getDefaultPool()); + batch.numElements = 1; + batch.notNull[0] = 1; + batch.data[0] = 1; + batch.nanoseconds[0] = 0; + const DataTypeDateTimeV2 type(6); + for (const auto kind : {orc::TIMESTAMP, orc::TIMESTAMP_INSTANT}) { + auto file_type = orc::createPrimitiveType(kind); + OrcDecodedColumnView view {.file_type = file_type.get(), + .batch = &batch, + .rows = 1, + .timestamp_context = &context}; + auto column = type.create_column(); + auto status = type.get_serde()->read_column_from_orc(*column, view); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(type.to_string(*column, 0), kind == orc::TIMESTAMP + ? "1969-12-31 19:00:01.000000" + : "1970-01-01 08:00:01.000000"); + } +} + +TEST(OrcTimestampDecodeContextTest, NestedViewsKeepContextAcrossStripeTimezoneChanges) { + OrcTimestampDecodeContext context(cctz::fixed_time_zone(cctz::seconds(8 * 3600))); + auto schema = orc::createStructType(); + schema->addStructField("plain", orc::createPrimitiveType(orc::TIMESTAMP)); + schema->addStructField("instant", orc::createPrimitiveType(orc::TIMESTAMP_INSTANT)); + orc::TimestampVectorBatch batch(1, *orc::getDefaultPool()); + batch.numElements = 1; + batch.notNull[0] = 1; + batch.data[0] = 1; + batch.nanoseconds[0] = 123; + const std::vector selected_rows {0}; + OrcDecodedColumnView parent {.file_type = schema.get(), + .selected_type = schema.get(), + .rows = 1, + .timestamp_context = &context}; + auto plain_view = orc_serde_utils::make_child_orc_view( + parent, schema->getSubtype(0), schema->getSubtype(0), &batch, 1, &selected_rows); + auto instant_view = orc_serde_utils::make_child_orc_view( + parent, schema->getSubtype(1), schema->getSubtype(1), &batch, 1, &selected_rows); + ASSERT_EQ(plain_view.timestamp_context, &context); + ASSERT_EQ(instant_view.timestamp_context, &context); + + const DataTypeTimeStampNs plain_type; + const DataTypeDateTimeV2 instant_type(6); + for (const std::string zone : {"America/New_York", "UTC"}) { + ASSERT_TRUE(cctz::load_time_zone(zone, &context.reader_timezone)); + auto plain_column = plain_type.create_column(); + auto status = plain_type.get_serde()->read_column_from_orc(*plain_column, plain_view); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ( + plain_type.to_string(*plain_column, 0), + zone == "UTC" ? "1970-01-01 00:00:01.000000123" : "1969-12-31 19:00:01.000000123"); + auto instant_column = instant_type.create_column(); + status = instant_type.get_serde()->read_column_from_orc(*instant_column, instant_view); + ASSERT_TRUE(status.ok()) << status; + // Changing a stripe's reader zone must never change an instant sibling's display zone. + EXPECT_EQ(instant_type.to_string(*instant_column, 0), "1970-01-01 08:00:01.000000"); + } +} + +} // namespace doris diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 1ca11ecb6963e5..95bd41d63e3f3a 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -62,6 +62,7 @@ #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "core/data_type/primitive_type.h" +#include "core/data_type_serde/orc_timestamp_decode_context.h" #include "core/value/timestamptz_value.h" #include "exprs/create_predicate_function.h" #include "exprs/runtime_filter_expr.h" @@ -10721,11 +10722,12 @@ Status decode_orc_timestamp_boundary(int64_t seconds, int64_t nanoseconds, bool batch.numElements = 1; batch.data[0] = seconds; batch.nanoseconds[0] = nanoseconds; + const OrcTimestampDecodeContext timestamp_context(timezone); OrcDecodedColumnView view {.file_type = type.get(), .selected_type = type.get(), .batch = &batch, .rows = 1, - .timezone = &timezone, + .timestamp_context = ×tamp_context, .enable_mapping_timestamp_tz = use_timestamp_tz}; if (use_timestamp_tz) { DataTypeTimeStampTz data_type(6); From 261b7b50e77dff6fde1e5feb3aaf5a41b7b24f62 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 14:11:46 +0800 Subject: [PATCH 11/13] [fix](be) Give the ORC timestamp read plan external linkage ### What problem does this PR solve? Related PR: #68934 Problem Summary: OrcReaderScanState has external linkage but embeds an OrcTimestampReadPlan from an anonymous namespace. GCC rejects this with -Werror=subobject-linkage, which the Clang build did not diagnose. Move the read plan into the same named namespace as the scan state and document the linkage requirement. The class implementation and SDK remain unchanged. ### Release note None ### Check List (For Author) - Test: Reproduced the error with GCC 15 on the production ORC unity translation unit; the same syntax check with CI warning flags passes after the fix. All 827 focused BE tests and the full ASAN BE build pass. clang-format 16 and clang-tidy checks pass. - Behavior changed: No - Does this need documentation: No --- be/src/format_v2/orc/orc_reader.cpp | 134 ++++++++++++++-------------- 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b6f374502fd8bb..c7585e0fbcfc48 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -333,72 +333,6 @@ bool is_virtual_column(format::LocalColumnId file_column_id) { return is_row_position_column(file_column_id) || is_global_rowid_column(file_column_id); } -// Keep SDK workarounds and their scan requirements in one plan. SerDes receive only the -// resulting decode context and never need to infer which reader strategy produced a batch. -class OrcTimestampReadPlan { -public: - void configure(const ::orc::Type& file_schema, - const std::vector& read_columns, - const cctz::time_zone& session_timezone) { - _context = OrcTimestampDecodeContext(session_timezone); - // An unprojected NANOS sibling can affect the SDK's ordinary timestamp decoding. - const bool needs_writer_timezone = - orc_serde_utils::has_nanosecond_timestamp(file_schema) && - std::ranges::any_of(read_columns, [&](const auto column_id) { - return !is_virtual_column(column_id) && - has_timestamp(*file_schema.getSubtype(column_id.value()), false); - }); - _timezone_source = - needs_writer_timezone ? TimezoneSource::STRIPE_WRITER : TimezoneSource::SESSION; - } - - bool requires_stripe_reader() const { - return _timezone_source == TimezoneSource::STRIPE_WRITER; - } - - bool requires_dense_decoding(const ::orc::Type& column_type) const { - // The pinned SDK's selected timestamp decoder checks a fraction at the wrong row index. - // Decode timestamp leaders densely while retaining lazy reads for other followers. - return requires_stripe_reader() && has_timestamp(column_type, true); - } - - Status configure_stripe(const ::orc::StripeInformation& stripe, - ::orc::RowReaderOptions* options) { - DORIS_CHECK(requires_stripe_reader()); - const auto& name = stripe.getWriterTimezone(); - const auto writer_timezone = name.empty() ? "GMT" : name; - if (!TimezoneUtils::find_cctz_time_zone(writer_timezone, _context.reader_timezone)) { - return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", - writer_timezone); - } - // Matching zones preserves the SDK's UTC carrier until its negative-fraction correction - // is complete. SerDes then apply historical offsets to that original second with cctz. - options->setTimezoneName(writer_timezone); - return Status::OK(); - } - - const OrcTimestampDecodeContext& decode_context() const { return _context; } - -private: - enum class TimezoneSource { SESSION, STRIPE_WRITER }; - - static bool has_timestamp(const ::orc::Type& type, bool include_instant) { - if (type.getKind() == ::orc::TIMESTAMP || - (include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) { - return true; - } - for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { - if (has_timestamp(*type.getSubtype(child), include_instant)) { - return true; - } - } - return false; - } - - TimezoneSource _timezone_source = TimezoneSource::SESSION; - OrcTimestampDecodeContext _context {cctz::utc_time_zone()}; -}; - format::ColumnDefinition nullable_global_rowid_column_definition() { auto field = format::global_rowid_column_definition(); field.type = make_nullable(field.type); @@ -801,6 +735,74 @@ Status find_projected_minmax_leaf(const ::orc::Type& root_type, } // namespace +// OrcReaderScanState has external linkage, so GCC requires its member types to stay +// outside the anonymous namespace (-Wsubobject-linkage). +// Keep SDK workarounds and their scan requirements in one plan. SerDes receive only the +// resulting decode context and never need to infer which reader strategy produced a batch. +class OrcTimestampReadPlan { +public: + void configure(const ::orc::Type& file_schema, + const std::vector& read_columns, + const cctz::time_zone& session_timezone) { + _context = OrcTimestampDecodeContext(session_timezone); + // An unprojected NANOS sibling can affect the SDK's ordinary timestamp decoding. + const bool needs_writer_timezone = + orc_serde_utils::has_nanosecond_timestamp(file_schema) && + std::ranges::any_of(read_columns, [&](const auto column_id) { + return !is_virtual_column(column_id) && + has_timestamp(*file_schema.getSubtype(column_id.value()), false); + }); + _timezone_source = + needs_writer_timezone ? TimezoneSource::STRIPE_WRITER : TimezoneSource::SESSION; + } + + bool requires_stripe_reader() const { + return _timezone_source == TimezoneSource::STRIPE_WRITER; + } + + bool requires_dense_decoding(const ::orc::Type& column_type) const { + // The pinned SDK's selected timestamp decoder checks a fraction at the wrong row index. + // Decode timestamp leaders densely while retaining lazy reads for other followers. + return requires_stripe_reader() && has_timestamp(column_type, true); + } + + Status configure_stripe(const ::orc::StripeInformation& stripe, + ::orc::RowReaderOptions* options) { + DORIS_CHECK(requires_stripe_reader()); + const auto& name = stripe.getWriterTimezone(); + const auto writer_timezone = name.empty() ? "GMT" : name; + if (!TimezoneUtils::find_cctz_time_zone(writer_timezone, _context.reader_timezone)) { + return Status::InvalidArgument("Unknown ORC stripe writer timezone: {}", + writer_timezone); + } + // Matching zones preserves the SDK's UTC carrier until its negative-fraction correction + // is complete. SerDes then apply historical offsets to that original second with cctz. + options->setTimezoneName(writer_timezone); + return Status::OK(); + } + + const OrcTimestampDecodeContext& decode_context() const { return _context; } + +private: + enum class TimezoneSource { SESSION, STRIPE_WRITER }; + + static bool has_timestamp(const ::orc::Type& type, bool include_instant) { + if (type.getKind() == ::orc::TIMESTAMP || + (include_instant && type.getKind() == ::orc::TIMESTAMP_INSTANT)) { + return true; + } + for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) { + if (has_timestamp(*type.getSubtype(child), include_instant)) { + return true; + } + } + return false; + } + + TimezoneSource _timezone_source = TimezoneSource::SESSION; + OrcTimestampDecodeContext _context {cctz::utc_time_zone()}; +}; + class OrcReader::OrcFilterImpl final : public ::orc::ORCFilter { public: explicit OrcFilterImpl(OrcReader* reader) : _reader(reader) {} From 1d617d19eebfaa940de306aee960113198393b51 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 13 Oct 2026 16:09:16 +0800 Subject: [PATCH 12/13] [fix](datalake) Preserve nanos schemas and safe Iceberg filters ### What problem does this PR solve? Related PR: #68934 Problem Summary: TVF schema discovery rounded nanosecond files before V2 decoding. Generated Iceberg overwrite/conflict predicates bypassed retained DATE-bound safeguards, and partial predicates under NOT could prune matching files. Historical ORC producer semantics, metadata MIN/MAX, dropped partition sources, per-stripe reader setup, and nullable Parquet nanos dispatch also needed correction. Route ORC/Parquet schema discovery through V2, require exact negated predicates, reject unsafe static overwrite, and preserve conservative conflict filtering. Keep Java/C++ historical carriers distinct, fall back from unsafe metadata aggregates, coalesce equal-zone stripes, and restore fused nullable decoding. SDK sources and legacy LOAD/V1 decoding are unchanged. ### Release note Preserve nanosecond precision in ORC/Parquet TVF schema discovery and correct affected Iceberg filtering and ORC timestamp reads. ### Check List (For Author) - Test: 425 FE unit tests; 949 BE unit tests; Iceberg nanosecond regression with real S3 ORC/Parquet TVF reads; BE/FE builds; FE Checkstyle; clang-format 16; clang-tidy; build hygiene; GCC 15 syntax check. - Behavior changed: Yes, preserve TVF nanos precision, reject unsafe static overwrites, and avoid incorrect pruning or historical timestamp results. - Does this need documentation: No, review corrections to the existing feature. --- .../data_type_datetimev2_serde.cpp | 9 +- .../data_type_timestamp_ns_serde.cpp | 2 +- .../orc_timestamp_decode_context.cpp | 6 +- .../orc_timestamp_decode_context.h | 7 +- be/src/format/CMakeLists.txt | 5 + be/src/format_v2/orc/orc_reader.cpp | 77 ++++++++-- be/src/format_v2/orc/orc_reader.h | 1 + .../reader/native/column_chunk_reader.cpp | 2 + be/src/format_v2/schema_reader.cpp | 66 +++++++++ be/src/format_v2/schema_reader.h | 39 +++++ be/src/service/internal_service.cpp | 22 ++- be/test/format_v2/orc/orc_reader_test.cpp | 138 +++++++++++++++++- .../GenerateJavaTimestampFixture.java | 52 +++++++ .../orc/test_data/java_timestamp_nanos.orc | Bin 0 -> 425 bytes .../format_v2/parquet/native_decoder_test.cpp | 31 ++++ .../format_v2/parquet/parquet_reader_test.cpp | 13 ++ .../iceberg/IcebergConnectorTransaction.java | 17 ++- .../iceberg/IcebergPartitionUtils.java | 4 +- .../iceberg/IcebergPredicateConverter.java | 27 +++- .../IcebergConnectorTransactionTest.java | 30 ++++ .../iceberg/IcebergScanPlanProviderTest.java | 28 +++- .../iceberg/IcebergTimestampNsTest.java | 49 +++++++ .../iceberg/test_iceberg_timestamp_ns.out | 14 ++ .../iceberg/test_iceberg_timestamp_ns.groovy | 15 ++ 24 files changed, 607 insertions(+), 47 deletions(-) create mode 100644 be/src/format_v2/schema_reader.cpp create mode 100644 be/src/format_v2/schema_reader.h create mode 100644 be/test/format_v2/orc/test_data/GenerateJavaTimestampFixture.java create mode 100644 be/test/format_v2/orc/test_data/java_timestamp_nanos.orc diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 786833599c39a8..36fc722e60d459 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -53,8 +53,7 @@ static const int64_t micro_to_nano_second = 1000; namespace { -Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view, - const cctz::time_zone& timezone) { +Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view) { const auto* orc_batch = dynamic_cast(orc_view.batch); if (orc_batch == nullptr) { return Status::InternalError("Unexpected ORC timestamp batch type {}", @@ -87,7 +86,8 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum // so rounding cannot jump across a daylight-saving gap or fold. status = orc_serde_utils::orc_timestamp_to_datetime( is_timestamp_instant ? timestamp.seconds : seconds, timestamp.microseconds, - timezone, !is_timestamp_instant && timestamp.carry, &value); + orc_view.timestamp_context->timezone_for(*orc_view.file_type, seconds), + !is_timestamp_instant && timestamp.carry, &value); if (!status.ok()) { data.resize(old_data_size); return status; @@ -1051,8 +1051,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { return Status::OK(); } - return decode_timestamp_orc_values(column, view, - view.timestamp_context->timezone_for(*view.file_type)); + return decode_timestamp_orc_values(column, view); } } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 5fd2c968fc11c5..deddf9ffc64655 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -480,7 +480,6 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows); DORIS_CHECK(view.file_type != nullptr); DORIS_CHECK(view.timestamp_context != nullptr); - const auto& timezone = view.timestamp_context->timezone_for(*view.file_type); for (size_t row = 0; row < rows; ++row) { const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows); if (orc_serde_utils::orc_row_is_null(batch, source)) { @@ -494,6 +493,7 @@ Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column, } DateV2Value civil; const int64_t seconds = batch.data[source]; + const auto timezone = view.timestamp_context->timezone_for(*view.file_type, seconds); auto status = orc_serde_utils::orc_timestamp_to_datetime(seconds, nanos / 1000, timezone, false, &civil); TimeStampNsValue value; diff --git a/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp b/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp index 53152b57670ecb..c42c7b148a3b14 100644 --- a/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp +++ b/be/src/core/data_type_serde/orc_timestamp_decode_context.cpp @@ -23,11 +23,15 @@ namespace doris { -const cctz::time_zone& OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type) const { +cctz::time_zone OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type, + int64_t seconds) const { const auto kind = file_type.getKind(); DORIS_CHECK(kind == orc::TIMESTAMP || kind == orc::TIMESTAMP_INSTANT); // Plain timestamps reconstruct the SDK reader's civil value. Instant-to-DATETIME conversion // uses the session zone, which must not follow a plain sibling's stripe writer timezone. + if (kind == orc::TIMESTAMP && writer_timezone) { + return writer_timezone(seconds); + } return kind == orc::TIMESTAMP ? reader_timezone : session_timezone; } diff --git a/be/src/core/data_type_serde/orc_timestamp_decode_context.h b/be/src/core/data_type_serde/orc_timestamp_decode_context.h index 228c781d0ba2f7..2a0609cd8ce96c 100644 --- a/be/src/core/data_type_serde/orc_timestamp_decode_context.h +++ b/be/src/core/data_type_serde/orc_timestamp_decode_context.h @@ -19,6 +19,8 @@ #include +#include + namespace orc { class Type; } @@ -30,7 +32,10 @@ struct OrcTimestampDecodeContext { explicit OrcTimestampDecodeContext(const cctz::time_zone& timezone) : reader_timezone(timezone), session_timezone(timezone) {} - const cctz::time_zone& timezone_for(const orc::Type& file_type) const; + cctz::time_zone timezone_for(const orc::Type& file_type, int64_t seconds = 0) const; + + // Java carriers follow the SDK writer-zone rules rather than cctz historical LMT. + std::function writer_timezone; cctz::time_zone reader_timezone; cctz::time_zone session_timezone; diff --git a/be/src/format/CMakeLists.txt b/be/src/format/CMakeLists.txt index 94b7c6bb6d1b4c..5ea7f2cce4e418 100644 --- a/be/src/format/CMakeLists.txt +++ b/be/src/format/CMakeLists.txt @@ -28,6 +28,11 @@ list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES}) add_library(Format STATIC ${SRC_FILES}) +# Read the pinned SDK timezone rules without changing SDK decoding or public interfaces. +target_include_directories(Format PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../../../contrib/apache-orc/c++/src + ${BUILD_DIR}/apache-orc/c++/src) + pch_reuse(Format) # Unity build scoped to format_v2/: the v2 readers are the maintained line and diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index c7585e0fbcfc48..74cdb27560ee90 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -766,7 +767,7 @@ class OrcTimestampReadPlan { return requires_stripe_reader() && has_timestamp(column_type, true); } - Status configure_stripe(const ::orc::StripeInformation& stripe, + Status configure_stripe(const ::orc::StripeInformation& stripe, ::orc::WriterId writer, ::orc::RowReaderOptions* options) { DORIS_CHECK(requires_stripe_reader()); const auto& name = stripe.getWriterTimezone(); @@ -778,6 +779,22 @@ class OrcTimestampReadPlan { // Matching zones preserves the SDK's UTC carrier until its negative-fraction correction // is complete. SerDes then apply historical offsets to that original second with cctz. options->setTimezoneName(writer_timezone); + if (writer == ::orc::ORC_JAVA_WRITER) { + // Java and C++ ancient carriers differ. Preserve the SDK Java writer rules + // instead of applying cctz LMT to a Java standard-offset carrier. + const auto* timezone = &::orc::getTimezoneByName(writer_timezone); + _context.writer_timezone = [timezone, + zones = std::unordered_map {}]( + int64_t seconds) mutable { + const auto offset = timezone->getVariant(seconds).gmtOffset; + auto it = zones.find(offset); + if (it == zones.end()) { + // Cache the few SDK offsets; fixed_time_zone takes a global cache lock. + it = zones.emplace(offset, cctz::fixed_time_zone(cctz::seconds(offset))).first; + } + return it->second; + }; + } return Status::OK(); } @@ -888,6 +905,8 @@ void OrcReader::_init_profile() { file_scan_profile::ensure_hierarchy(_profile); _orc_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); + _orc_profile.row_readers_created = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcRowReadersCreated", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -1786,20 +1805,33 @@ Status OrcReader::_prepare_timestamp_stripe_ranges() { try { uint64_t first_row = 0; size_t selected = 0; + std::string previous_timezone; for (uint64_t index = 0; index <= stripes.back(); ++index) { const auto stripe = _state->reader->getStripe(index); const auto end_row = first_row + stripe->getNumberOfRows(); if (index == stripes[selected]) { - // A RowReader has one timezone, whereas writer timezones belong to stripes. - // Read footer timezone metadata only when entering the selected stripe. - _state->selected_stripe_ranges.push_back({ - .first_stripe = index, - .last_stripe = index + 1, - .offset = stripe->getOffset(), - .length = stripe->getLength(), - .first_row = first_row, - .end_row = end_row, - }); + const auto& name = stripe->getWriterTimezone(); + const std::string timezone = name.empty() ? "GMT" : name; + auto& ranges = _state->selected_stripe_ranges; + // Each SDK RowReader scans the entire stripe directory. Reuse it across + // adjacent stripes with the same timezone to avoid quadratic setup work. + if (!ranges.empty() && ranges.back().last_stripe == index && + previous_timezone == timezone) { + auto& range = ranges.back(); + range.last_stripe = index + 1; + range.length = stripe->getOffset() + stripe->getLength() - range.offset; + range.end_row = end_row; + } else { + ranges.push_back({ + .first_stripe = index, + .last_stripe = index + 1, + .offset = stripe->getOffset(), + .length = stripe->getLength(), + .first_row = first_row, + .end_row = end_row, + }); + } + previous_timezone = timezone; ++selected; } first_row = end_row; @@ -1820,8 +1852,8 @@ Status OrcReader::_apply_current_stripe_range() { if (_state->timestamp_plan.requires_stripe_reader()) { try { const auto stripe = _state->reader->getStripe(stripe_range.first_stripe); - RETURN_IF_ERROR( - _state->timestamp_plan.configure_stripe(*stripe, &_state->row_reader_options)); + RETURN_IF_ERROR(_state->timestamp_plan.configure_stripe( + *stripe, _state->reader->getWriterId(), &_state->row_reader_options)); } catch (const std::exception& e) { if (is_orc_stop(_io_ctx.get(), e)) { return Status::EndOfFile("stop"); @@ -1854,6 +1886,9 @@ Status OrcReader::_create_row_reader() { _state->row_reader = _state->reader->createRowReader( _state->row_reader_options, _state->orc_lazy_read_enabled ? _orc_filter.get() : nullptr); + if (_profile != nullptr) { + COUNTER_UPDATE(_orc_profile.row_readers_created, 1); + } _state->selected_type = &_state->row_reader->getSelectedType(); DORIS_CHECK(_state->selected_type->getKind() == ::orc::TypeKind::STRUCT); // Row-id fetch seeks before every read; a one-row batch preserves exact selection instead @@ -2426,6 +2461,22 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque } std::unique_ptr<::orc::StripeStatistics> stripe_statistics; try { + if (leaf_type->getKind() == ::orc::TIMESTAMP && + orc_serde_utils::has_nanosecond_timestamp(*_state->root_type) && + _state->reader->getWriterId() == ::orc::ORC_CPP_WRITER) { + // Named-zone C++ statistics use SDK historical offsets, whereas rows use cctz. + // Preserve exact metadata aggregates for fixed-zone writers; otherwise scan rows. + const auto stripe = _state->reader->getStripe(stripe_index); + const auto& name = stripe->getWriterTimezone(); + cctz::time_zone writer_zone; + int32_t offset = 0; + if (!TimezoneUtils::find_cctz_time_zone(name.empty() ? "GMT" : name, + writer_zone) || + !TimezoneUtils::try_get_fixed_offset_seconds(writer_zone, &offset)) { + return Status::NotSupported( + "ORC C++ named-zone mixed nanos timestamps require row min/max"); + } + } stripe_statistics = _state->reader->getStripeStatistics(stripe_index); } catch (const std::exception& e) { return Status::InternalError("Failed to read ORC stripe statistics {}: {}", diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index 65ad24f7ee40aa..0a10aad3f54b0e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -73,6 +73,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* row_readers_created = nullptr; RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index f08791674d5951..2a6467b9b3584c 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -592,6 +592,8 @@ bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { VISIT_COLUMN(ColumnUUID) VISIT_COLUMN(ColumnTimeV2) VISIT_COLUMN(ColumnTimeStampTz) + // Nanos columns also support in-place NULL expansion; do not decode once per null run. + VISIT_COLUMN(ColumnTimeStampNs) VISIT_COLUMN(ColumnOffset32) VISIT_COLUMN(ColumnOffset64) VISIT_COLUMN(ColumnDecimal32) diff --git a/be/src/format_v2/schema_reader.cpp b/be/src/format_v2/schema_reader.cpp new file mode 100644 index 00000000000000..6bc0233fbde227 --- /dev/null +++ b/be/src/format_v2/schema_reader.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/schema_reader.h" + +#include "format_v2/orc/orc_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "io/file_factory.h" +#include "runtime/runtime_state.h" + +namespace doris::format { + +Status fetch_file_schema(const TFileScanRangeParams& params, const TFileRangeDesc& range, + const std::shared_ptr& io_ctx, RuntimeProfile* profile, + std::vector* schema) { + auto properties = std::make_shared(); + // Schema RPCs carry the filesystem kind on the range, not necessarily in scan params. + properties->system_type = range.__isset.file_type ? range.file_type : params.file_type; + properties->properties = params.properties; + properties->hdfs_params = params.hdfs_params; + properties->broker_addresses = params.broker_addresses; + auto file = std::make_unique(); + file->path = range.path; + file->file_size = range.__isset.file_size ? range.file_size : -1; + file->mtime = range.__isset.modification_time ? range.modification_time : 0; + file->fs_name = range.fs_name; + if (range.__isset.file_cache_admission) { + file->file_cache_admission = range.file_cache_admission; + } + const bool timestamp_tz = + params.__isset.enable_mapping_timestamp_tz && params.enable_mapping_timestamp_tz; + const bool varbinary = + params.__isset.enable_mapping_varbinary && params.enable_mapping_varbinary; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + std::unique_ptr reader; + // TVF targets must retain file precision before column mapping; a microsecond schema + // irreversibly rounds nanos even when the subsequent row scan uses the V2 reader. + if (params.format_type == TFileFormatType::FORMAT_PARQUET) { + reader = std::make_unique(properties, file, io_ctx, profile, + std::nullopt, timestamp_tz, varbinary); + } else if (params.format_type == TFileFormatType::FORMAT_ORC) { + reader = std::make_unique(properties, file, io_ctx, profile, std::nullopt, + timestamp_tz, varbinary); + } else { + return Status::NotSupported("V2 schema inference requires Parquet or ORC"); + } + RETURN_IF_ERROR(reader->init(&state)); + RETURN_IF_ERROR(reader->get_schema(schema)); + return reader->close(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/schema_reader.h b/be/src/format_v2/schema_reader.h new file mode 100644 index 00000000000000..ad92c557ec9e0a --- /dev/null +++ b/be/src/format_v2/schema_reader.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" + +namespace doris { +class RuntimeProfile; +class TFileScanRangeParams; +class TFileRangeDesc; +namespace io { +struct IOContext; +} +namespace format { +struct ColumnDefinition; + +Status fetch_file_schema(const TFileScanRangeParams& params, const TFileRangeDesc& range, + const std::shared_ptr& io_ctx, RuntimeProfile* profile, + std::vector* schema); +} // namespace format +} // namespace doris diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 3ff14bdf8ce5fc..a2014b7bb3d452 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -74,9 +74,9 @@ #include "format/generic_reader.h" #include "format/jni/jni_reader.h" #include "format/json/new_json_reader.h" -#include "format/orc/vorc_reader.h" -#include "format/parquet/vparquet_reader.h" #include "format/text/text_reader.h" +#include "format_v2/column_data.h" +#include "format_v2/schema_reader.h" #include "io/fs/local_file_system.h" #include "io/fs/stream_load_pipe.h" #include "io/io_common.h" @@ -908,13 +908,19 @@ void PInternalService::fetch_table_schema(google::protobuf::RpcController* contr file_slots, fetch_schema_batch_size, io_ctx.get()); break; } - case TFileFormatType::FORMAT_PARQUET: { - reader = ParquetReader::create_unique(params, range, io_ctx, nullptr); - break; - } + case TFileFormatType::FORMAT_PARQUET: case TFileFormatType::FORMAT_ORC: { - reader = OrcReader::create_unique(params, range, fetch_schema_batch_size, "", io_ctx); - break; + std::vector schema; + st = format::fetch_file_schema(params, range, io_ctx, profile.get(), &schema); + if (st.ok()) { + result->set_column_nums(schema.size()); + for (const auto& column : schema) { + result->add_column_names(column.name); + column.type->to_protobuf(result->add_column_types()); + } + } + st.to_protobuf(result->mutable_status()); + return; } case TFileFormatType::FORMAT_JSON: { reader = NewJsonReader::create_unique(profile.get(), params, range, file_slots, diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 95bd41d63e3f3a..a2cb3123af7086 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -77,6 +77,7 @@ #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_profile.h" +#include "format_v2/schema_reader.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -10580,6 +10581,107 @@ TEST_F(NewOrcReaderTest, NanosecondFilesPreserveNegativeFractionsAcrossWriterTim } } +TEST_F(NewOrcReaderTest, AdjacentNanosStripesReuseOneRowReader) { + const auto file = (_test_dir / "many_nanos_stripes.orc").string(); + auto type = ::orc::createStructType(); + auto ns = ::orc::createPrimitiveType(::orc::TIMESTAMP); + ns->setAttribute("iceberg.timestamp-unit", "NANOS"); + type->addStructField("ns", std::move(ns)); + MemoryOutputStream stream(1024 * 1024); + ::orc::WriterOptions options; + options.setTimezoneName("UTC"); + options.setStripeSize(0); + auto writer = ::orc::createWriter(*type, &stream, options); + constexpr size_t STRIPES = 64; + for (size_t i = 0; i < STRIPES; ++i) { + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + root.numElements = 1; + auto& ts = dynamic_cast<::orc::TimestampVectorBatch&>(*root.fields[0]); + ts.numElements = 1; + ts.data[0] = 0; + ts.nanoseconds[0] = static_cast(i); + writer->add(*batch); + } + writer->close(); + { + std::ofstream out(file, std::ios::binary); + out.write(stream.getData(), static_cast(stream.getLength())); + } + ASSERT_EQ(get_orc_stripe_count(file), STRIPES); + RuntimeProfile profile("many_stripes"); + auto reader = create_reader_for_path(file, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + bool eof = false; + size_t total = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto& values = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + for (size_t i = 0; i < rows; ++i) { + EXPECT_EQ(values.get_element(i).epoch_nanos(), total + i); + } + total += rows; + } + EXPECT_EQ(total, STRIPES); + EXPECT_EQ(profile.get_counter("OrcRowReadersCreated")->value(), 1); +} + +TEST_F(NewOrcReaderTest, JavaNanosSiblingPreservesHistoricalCivilTime) { + const auto file = find_repo_file("be/test/format_v2/orc/test_data/java_timestamp_nanos.orc"); + const std::array plain {"1800-01-01 00:00:00.123457", + "1969-12-31 23:59:59.500000", + "1970-01-01 00:00:00.000000"}; + const std::array nanos {"1800-01-01 00:00:00.123456789", + "1969-12-31 23:59:59.500000123", + "1970-01-01 00:00:00.000000001"}; + for (const auto& zone : {"UTC", "Asia/Shanghai", "America/New_York"}) { + for (bool project_ns : {false, true}) { + auto reader = create_reader_for_path(file); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone(zone); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + if (project_ns) { + request->non_predicate_columns.push_back(field_projection(1)); + } else { + schema.resize(1); + } + ASSERT_TRUE(reader->open(request).ok()); + size_t row = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + for (size_t i = 0; i < rows; ++i, ++row) { + ASSERT_LT(row, plain.size()); + EXPECT_EQ(schema[0].type->to_string(*block.get_by_position(0).column, i), + plain[row]); + if (project_ns) { + EXPECT_EQ(schema[1].type->to_string(*block.get_by_position(1).column, i), + nanos[row]); + } + } + } + EXPECT_EQ(row, plain.size()); + } + } +} + TEST_F(NewOrcReaderTest, NanosecondFilesPreserveHistoricalWriterCivilTime) { const auto file_path = (_test_dir / "timestamp_ns_historical.orc").string(); auto type = ::orc::createStructType(); @@ -10599,15 +10701,15 @@ TEST_F(NewOrcReaderTest, NanosecondFilesPreserveHistoricalWriterCivilTime) { // The SDK accepts UTC seconds. These cover the historical LMT offset, the second // preceding a DST fold, and different fractions at the same negative epoch second. const std::array seconds {-5364644638, -5767201, 1, 1}; - const std::array nanos {0, 500000000, 500000123, 1}; + const std::array nanos {123456789, 500000000, 500000123, 1}; const std::array expected_ns { - "1800-01-01 00:00:00.000000000", "1969-10-26 01:59:59.500000000", + "1800-01-01 00:00:00.123456789", "1969-10-26 01:59:59.500000000", "1969-12-31 19:00:01.500000123", "1969-12-31 19:00:01.000000001"}; const std::array expected_plain { - "1800-01-01 00:00:00.000000", "1969-10-26 01:59:59.500000", + "1800-01-01 00:00:00.123457", "1969-10-26 01:59:59.500000", "1969-12-31 19:00:01.500000", "1969-12-31 19:00:01.000000"}; const std::array expected_instant { - "1800-01-01 12:56:02.000000", "1969-10-26 13:59:59.500000", + "1800-01-01 12:56:02.123457", "1969-10-26 13:59:59.500000", "1970-01-01 08:00:01.500000", "1970-01-01 08:00:01.000000"}; for (size_t start = 0; start < seconds.size(); start += 2) { auto batch = writer->createRowBatch(2); @@ -10632,11 +10734,36 @@ TEST_F(NewOrcReaderTest, NanosecondFilesPreserveHistoricalWriterCivilTime) { out.write(stream.getData(), static_cast(stream.getLength())); } ASSERT_EQ(get_orc_stripe_count(file_path), 2); + TFileScanRangeParams params; + params.__set_file_type(TFileType::FILE_S3); + params.__set_format_type(TFileFormatType::FORMAT_ORC); + TFileRangeDesc range; + range.__set_file_type(TFileType::FILE_LOCAL); + range.__set_path(file_path); + std::vector tvf_schema; + ASSERT_TRUE(format::fetch_file_schema(params, range, std::make_shared(), nullptr, + &tvf_schema) + .ok()); + ASSERT_EQ(tvf_schema.size(), 4); + EXPECT_EQ(tvf_schema[1].type->get_primitive_type(), TYPE_TIMESTAMP_NS); + auto aggregate_reader = create_reader_for_path(file_path); + RuntimeState aggregate_state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(aggregate_reader->init(&aggregate_state).ok()); + auto plain_request = std::make_shared(); + plain_request->non_predicate_columns = {field_projection(2)}; + ASSERT_TRUE(aggregate_reader->open(plain_request).ok()); + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back({.projection = field_projection(2)}); + format::FileAggregateResult aggregate_result; + EXPECT_TRUE(aggregate_reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); for (bool project_ns : {false, true}) { for (int mode : {0, 1, 2, 3}) { const bool lazy = mode == 1 || mode == 3; for (int threshold : {-1, 0, 2}) { - auto reader = create_reader_for_path(file_path); + RuntimeProfile profile("timestamp_stripe_ranges"); + auto reader = create_reader_for_path(file_path, &profile); TQueryOptions query_options; query_options.__set_enable_orc_lazy_mat(lazy); RuntimeState state {query_options, TQueryGlobals()}; @@ -10704,6 +10831,7 @@ TEST_F(NewOrcReaderTest, NanosecondFilesPreserveHistoricalWriterCivilTime) { } } EXPECT_EQ(output_row, seconds.size()); + EXPECT_EQ(profile.get_counter("OrcRowReadersCreated")->value(), 1); const auto& stats = reader->reader_statistics(); EXPECT_EQ(stats.lazy_read_filtered_rows + (lazy ? stats.filtered_group_rows : 0), lazy ? threshold + 1 : 0); diff --git a/be/test/format_v2/orc/test_data/GenerateJavaTimestampFixture.java b/be/test/format_v2/orc/test_data/GenerateJavaTimestampFixture.java new file mode 100644 index 00000000000000..b011bfa7115513 --- /dev/null +++ b/be/test/format_v2/orc/test_data/GenerateJavaTimestampFixture.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.Timestamp; +import java.util.TimeZone; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.orc.OrcFile; +import org.apache.orc.TypeDescription; +import org.apache.orc.Writer; + +// Generate with ORC 1.9.0, hadoop-client-{api,runtime} 3.3.5, hive-storage-api 2.8.1, +// and protobuf-java 4.34.1 on Java 17. +// The hidden NANOS sibling exercises the matching-writer-zone decode path. +public class GenerateJavaTimestampFixture { + public static void main(String[] args) throws Exception { + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + TypeDescription schema = TypeDescription.fromString("struct"); + schema.getChildren().get(1).setAttribute("iceberg.timestamp-unit", "NANOS"); + Configuration conf = new Configuration(); + conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + String[] values = {"1800-01-01 00:00:00.123456789", "1969-12-31 23:59:59.500000123", + "1970-01-01 00:00:00.000000001"}; + try (Writer writer = OrcFile.createWriter(new Path(args[0]), + OrcFile.writerOptions(conf).setSchema(schema).overwrite(true))) { + VectorizedRowBatch batch = schema.createRowBatch(); + batch.size = values.length; + for (int row = 0; row < values.length; row++) { + for (int col = 0; col < 2; col++) { + ((TimestampColumnVector) batch.cols[col]).set(row, Timestamp.valueOf(values[row])); + } + } + writer.addRowBatch(batch); + } + } +} diff --git a/be/test/format_v2/orc/test_data/java_timestamp_nanos.orc b/be/test/format_v2/orc/test_data/java_timestamp_nanos.orc new file mode 100644 index 0000000000000000000000000000000000000000..385ff055deba2fda51dfdff48494649130e1d3fe GIT binary patch literal 425 zcmeYdau#G@;9?VE;b0D6aAIJ19P}V5nSoi+kS|T2vGZK<_xn0`4lX&cpzv+;7RF!( zW}?(6GcZ&zF>9Qi^Os?zrwk*Ikp3^rz);3ybuVMZyKLnC%mT)Z8>TcQ=Vb+^6)oBqpH*3t zQWt2(;0H8d)1#z^3493;COt^_CN$xNP5J+`hh%-Yc%-h|c}N?u8n~=@G>wsgg^}TL zf>^}~jiW~o>ImpOT>43)P%`1W~v{Z629ihk%KFjoH>}OdR}$f z%sI2BL`_*2K56FcsF+DJq8CKZnjWcicN9=R242(zWX=3 z)p~}Agvq+v31$gg2^|4PbsrlXJm0L?WY~E%;C#?|V5qb*3NT7EGzcg$G4N<;8E`gz LVP^IZa+Ux9T{Ee^ literal 0 HcmV?d00001 diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index af4a17d1b8a418..1dc88ace240d90 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -37,6 +37,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/data_type_serde/parquet_timestamp.h" @@ -1331,6 +1332,36 @@ TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhys EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); } +TEST(ParquetV2NativeDecoderTest, NullableNanosSelectionBatchesPhysicalPayload) { + const std::vector nanos {-1, 123456789, 987654321, 1, 999999999}; + const std::vector ids {0, 1, 2, 3, 4}; + const std::vector null_runs(9, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS}; + for (bool dictionary : {false, true}) { + DataTypeTimeStampNs type; + auto column = type.create_column(); + NullMap nulls; + ColumnChunkReaderStatistics statistics; + auto status = dictionary ? materialize_selected_dictionary_int64( + nanos, ids, 9, null_runs, filter, type, &column, &nulls, + &statistics, context) + : materialize_selected_plain_int64(nanos, 9, null_runs, filter, + type, &column, &nulls, + &statistics, context); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column->size(), 6); + const auto& values = assert_cast(*column); + EXPECT_EQ(values.get_element(0).epoch_nanos(), -1); + EXPECT_EQ(values.get_element(2).epoch_nanos(), 123456789); + EXPECT_EQ(values.get_element(4).epoch_nanos(), 1); + EXPECT_EQ(nulls, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + } +} + TEST(ParquetV2NativeDecoderTest, NegativeNanosFloorAcrossPlainAndDictionaryTimestamps) { const std::vector nanos {-1, -999, -1000, -1001, 0, 1001}; const std::vector dictionary_ids {0, 1, 2, 3, 4, 5}; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 37370b64218874..2a5061323e9c73 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -72,6 +72,7 @@ #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/schema_projection.h" +#include "format_v2/schema_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/fs/file_meta_cache.h" @@ -3028,6 +3029,18 @@ TEST_F(NewParquetReaderTest, NanosecondsSurviveWallClockProjectionAndDictionary) ASSERT_TRUE(reader->init(&state).ok()); std::vector schema; ASSERT_TRUE(reader->get_schema(&schema).ok()); + TFileScanRangeParams params; + params.__set_file_type(TFileType::FILE_S3); + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + TFileRangeDesc range; + range.__set_file_type(TFileType::FILE_LOCAL); + range.__set_path(_file_path); + std::vector tvf_schema; + ASSERT_TRUE(format::fetch_file_schema(params, range, std::make_shared(), + nullptr, &tvf_schema) + .ok()); + ASSERT_EQ(tvf_schema.size(), 1); + EXPECT_EQ(tvf_schema[0].type->get_primitive_type(), TYPE_TIMESTAMP_NS); ASSERT_EQ(1, schema.size()); ASSERT_EQ(TYPE_TIMESTAMP_NS, schema[0].type->get_primitive_type()); auto projection = field_projection(0); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index 8f38b3186062ed..fd8706c3bada36 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -889,6 +889,7 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa return Expressions.alwaysTrue(); } + Set unsafeNanosFields = IcebergPredicateConverter.unsafeNanosFields(transaction.table(), schema); List predicates = new ArrayList<>(); Set unmatchedFields = new HashSet<>(staticPartitions.keySet()); for (PartitionField field : spec.fields()) { @@ -903,6 +904,12 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa } Object partitionValue = IcebergPartitionUtils.parsePartitionValueFromString( partitionValueStr, sourceField.type(), zone); + // Dropping an unsafe deletion predicate would widen the overwrite to other partitions. + if (partitionValue != null && unsafeNanosFields.contains(sourceField.fieldId())) { + throw new DorisConnectorException( + "Cannot overwrite timestamp_ns partition with historical DATE bounds: " + + sourceField.name()); + } predicates.add(identityPartitionPredicate(sourceField, partitionValue)); } } @@ -1181,6 +1188,7 @@ private Optional buildConflictDetectionFilter(Table icebergTable, } Schema schema = icebergTable.schema(); + Set unsafeNanosFields = IcebergPredicateConverter.unsafeNanosFields(icebergTable, schema); int currentSpecId = spec.specId(); Expression combined = null; @@ -1197,7 +1205,8 @@ private Optional buildConflictDetectionFilter(Table icebergTable, return Optional.empty(); } - Expression partitionExpr = buildIdentityPartitionExpression(spec, schema, partitionValues); + Expression partitionExpr = buildIdentityPartitionExpression( + spec, schema, partitionValues, unsafeNanosFields); if (partitionExpr == null) { return Optional.empty(); } @@ -1216,7 +1225,7 @@ private boolean areAllIdentityPartitions(PartitionSpec spec) { } private Expression buildIdentityPartitionExpression(PartitionSpec spec, Schema schema, - List partitionValues) { + List partitionValues, Set unsafeNanosFields) { Expression expression = null; List fields = spec.fields(); for (int i = 0; i < fields.size(); i++) { @@ -1230,6 +1239,10 @@ private Expression buildIdentityPartitionExpression(PartitionSpec spec, Schema s valueStr = null; } Object value = IcebergPartitionUtils.parsePartitionValueFromString(valueStr, sourceField.type(), zone); + // Conflict detection may widen, but must never miss files with retained DATE metrics. + if (value != null && unsafeNanosFields.contains(sourceField.fieldId())) { + continue; + } Expression predicate = identityPartitionPredicate(sourceField, value); expression = expression == null ? predicate : Expressions.and(expression, predicate); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index fdd540a0bd373a..46f7659f698f55 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -828,7 +828,9 @@ static boolean isValidRelatedTable(Table table) { return false; } PartitionField partitionField = fields.get(0); - if (spec.schema().findType(partitionField.sourceId()).typeId() == Type.TypeID.TIMESTAMP_NANO) { + Type sourceType = spec.schema().findType(partitionField.sourceId()); + // Historical void partition fields can outlive their dropped source column. + if (sourceType == null || sourceType.typeId() == Type.TypeID.TIMESTAMP_NANO) { // Calendar boundaries and the NULL sentinel can exceed TIMESTAMP_NS's domain. // Leave pruning to Iceberg instead of building invalid Doris RANGE boundaries. return false; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java index 2c059d64cd0b7e..3f15bc808093cb 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java @@ -115,6 +115,7 @@ public class IcebergPredicateConverter { // so strictly all-or-nothing/precise) -- mirrors IcebergNereidsUtils.convertNereidsToIcebergExpression. The // shared leaf helpers (getPushdownField / extractIcebergLiteral / toMicros / checkConversion) are reused. private final Mode mode; + private final boolean requireExact; /** Conversion matrix selector. See the field comment and the class javadoc. */ public enum Mode { SCAN, CONFLICT, REWRITE } @@ -129,6 +130,11 @@ public IcebergPredicateConverter(Schema schema, ZoneId sessionZone, boolean conf } public IcebergPredicateConverter(Schema schema, ZoneId sessionZone, Mode mode) { + this(schema, sessionZone, mode, false); + } + + private IcebergPredicateConverter(Schema schema, ZoneId sessionZone, Mode mode, boolean requireExact) { + this.requireExact = requireExact; this.schema = schema; this.sessionZone = sessionZone == null ? ZoneOffset.UTC : sessionZone; this.mode = mode; @@ -136,6 +142,12 @@ public IcebergPredicateConverter(Schema schema, ZoneId sessionZone, Mode mode) { public static IcebergPredicateConverter forTable(Table table, Schema schema, ZoneId zone, Mode mode) { IcebergPredicateConverter converter = new IcebergPredicateConverter(schema, zone, mode); + converter.unsafeNanosFields.addAll(unsafeNanosFields(table, schema)); + return converter; + } + + static Set unsafeNanosFields(Table table, Schema schema) { + Set unsafeFields = new LinkedHashSet<>(); Map history = table.schemas(); // Iceberg allocates schema IDs from zero. Gaps mean cleanup removed type history, not // necessarily its data files; without that history any nanos field may have DATE bounds. @@ -147,18 +159,18 @@ public static IcebergPredicateConverter forTable(Table table, Schema schema, Zon continue; } if (incompleteHistory) { - converter.unsafeNanosFields.add(field.fieldId()); + unsafeFields.add(field.fieldId()); continue; } for (Schema historical : history.values()) { Type oldType = historical.findType(field.fieldId()); if (oldType != null && oldType.typeId() == TypeID.DATE) { - converter.unsafeNanosFields.add(field.fieldId()); + unsafeFields.add(field.fieldId()); break; } } } - return converter; + return unsafeFields; } /** @@ -241,7 +253,7 @@ private Expression buildAnd(ConnectorAnd and) { for (ConnectorExpression child : and.getConjuncts()) { Expression c = convertSingle(child); if (c == null) { - if (mode == Mode.REWRITE) { + if (mode == Mode.REWRITE || requireExact) { return null; } continue; @@ -265,7 +277,10 @@ private Expression buildOr(ConnectorOr or) { } private Expression buildNot(ConnectorNot not) { - Expression child = convertSingle(not.getOperand()); + // Negating a widened operand narrows the result and can permanently prune matching files. + IcebergPredicateConverter exact = new IcebergPredicateConverter(schema, sessionZone, mode, true); + exact.unsafeNanosFields.addAll(unsafeNanosFields); + Expression child = exact.convertSingle(not.getOperand()); return child == null ? null : Expressions.not(child); } @@ -795,7 +810,7 @@ private Expression checkConversion(Expression expression) { Expression right = checkConversion(andExpr.right()); if (left != null && right != null) { return andExpr; - } else if (mode == Mode.REWRITE) { + } else if (mode == Mode.REWRITE || requireExact) { // all-or-nothing: a single unbindable arm fails the whole AND (no silent widen for rewrite). return null; } else if (left != null) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 9c504ed00746f6..d1160b17d51ec9 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -889,6 +889,36 @@ public void overwriteStaticPartitionUsesRowFilter() { Assertions.assertEquals("1", snap.summary().get("added-data-files")); } + @Test + public void promotedDateStaticOverwriteRejectsUnsafeRetainedBounds() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.DateType.get())); + Table table = catalog.createTable(id, schema, PartitionSpec.unpartitioned(), props("format-version", "3")); + Map bounds = Collections.singletonMap(1, + org.apache.iceberg.types.Conversions.toByteBuffer(Types.DateType.get(), 19724)); + table.newAppend().appendFile(DataFiles.builder(table.spec()) + .withPath("s3://bucket/retained-date.parquet").withFileSizeInBytes(100).withRecordCount(1) + .withMetrics(new org.apache.iceberg.Metrics(1L, null, null, null, null, bounds, bounds)) + .withFormat(FileFormat.PARQUET).build()).commit(); + org.apache.iceberg.TableOperations ops = ((org.apache.iceberg.HasTableOperations) table).operations(); + org.apache.iceberg.TableMetadata base = ops.current(); + ops.commit(base, org.apache.iceberg.TableMetadata.buildFrom(base).setCurrentSchema(new Schema( + Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())), 1).build()); + table.refresh(); + table.updateSpec().addField("ts").commit(); + long snapshotId = table.currentSnapshot().snapshotId(); + IcebergConnectorTransaction txn = txnFor(opsReturning(table), new RecordingConnectorContext()); + txn.beginWrite(SESSION, "db1", "t1", overwriteStaticCtx(table, Collections.singletonMap("ts", "2024-01-02"))); + txn.addCommitData(commitBytes(dataFileItem("s3://bucket/replacement.parquet", 1L, 100L, + Collections.singletonList("2024-01-02 00:00:00.000000000")))); + // DATE metrics are days, so silently committing a nanos equality would retain the old file. + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, txn::commit); + Assertions.assertTrue(ex.getMessage().contains("historical DATE bounds"), ex.getMessage()); + table.refresh(); + Assertions.assertEquals(snapshotId, table.currentSnapshot().snapshotId()); + } + @Test public void dateOnlyNanosecondStaticOverwriteKeepsAdjacentPartition() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index a42e4ec1766c3f..77d39888fdfb33 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -27,6 +27,7 @@ import org.apache.doris.connector.spi.pushdown.ConnectorComparison; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.pushdown.ConnectorNot; import org.apache.doris.connector.spi.scan.ConnectorScanRange; import org.apache.doris.connector.spi.scan.ConnectorScanRequest; import org.apache.doris.connector.spi.scan.ConnectorSplitSource; @@ -4047,11 +4048,13 @@ public void manifestCacheUsesLatestSchemaForSchemaOnlyMvccPin() throws IOExcepti @Test public void promotedDateBoundsDoNotPruneNanosecondMatches() { - Schema oldSchema = new Schema(Types.NestedField.optional(1, "ts", Types.DateType.get())); + Schema oldSchema = new Schema(Types.NestedField.optional(1, "ts", Types.DateType.get()), + Types.NestedField.optional(2, "id", Types.IntegerType.get())); Table table = createTable("promoted_date", oldSchema, PartitionSpec.unpartitioned(), Collections.singletonMap(TableProperties.FORMAT_VERSION, "3")); - Map bounds = Collections.singletonMap( - 1, Conversions.toByteBuffer(Types.DateType.get(), 19724)); + Map bounds = new HashMap<>(); + bounds.put(1, Conversions.toByteBuffer(Types.DateType.get(), 19724)); + bounds.put(2, Conversions.toByteBuffer(Types.IntegerType.get(), 1)); table.newAppend().appendFile(DataFiles.builder(table.spec()) .withPath("/data/retained-date.parquet").withFileSizeInBytes(100).withRecordCount(1) .withMetrics(new Metrics(1L, null, null, null, null, bounds, bounds)) @@ -4062,7 +4065,8 @@ public void promotedDateBoundsDoNotPruneNanosecondMatches() { operations.commit(base, org.apache.iceberg.TableMetadata.buildFrom(base) .setCurrentSchema(new Schema( Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone()), - Types.NestedField.optional(2, "fresh", Types.TimestampNanoType.withoutZone())), 2).build()); + Types.NestedField.optional(2, "id", Types.IntegerType.get()), + Types.NestedField.optional(3, "fresh", Types.TimestampNanoType.withoutZone())), 3).build()); table.refresh(); ConnectorExpression filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), @@ -4103,6 +4107,22 @@ public void promotedDateBoundsDoNotPruneNanosecondMatches() { Assertions.assertEquals(sortedPaths(sdk), sortedPaths(cached)); Assertions.assertTrue(cache.size() > 0); Assertions.assertEquals(0L, cache.takeStats("q")[2], "must exercise cached planning without fallback"); + ConnectorExpression negated = new ConnectorNot(new ConnectorAnd(Arrays.asList( + new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), + new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-03")), + new ConnectorComparison(ConnectorComparison.Operator.GT, + new ConnectorColumnRef("id", ConnectorType.of("INT")), + new ConnectorLiteral(ConnectorType.of("INT"), 0))))); + ConnectorScanRequest notRequest = ConnectorScanRequest.builder( + new IcebergTableHandle("db1", "promoted_date"), Collections.emptyList()) + .filter(Optional.of(negated)).build(); + // The retained Jan 2 / id=1 file matches NOT(ts=Jan 3 AND id>0). + Assertions.assertEquals(2, new IcebergScanPlanProvider( + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) + .planScan(emptySession(), notRequest).size()); + Assertions.assertEquals(2, manifestProvider(manifestCacheProps(), table, cache) + .planScan(emptySession(), notRequest).size()); } // --- T09: vended credentials (extractVendedToken + static/vended location.* + URI threading) --- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java index 2a141c13659e0e..a69d4a6faea7ed 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimestampNsTest.java @@ -18,9 +18,11 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorAnd; import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; import org.apache.doris.connector.spi.pushdown.ConnectorComparison; import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.pushdown.ConnectorNot; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -41,6 +43,53 @@ import java.util.Optional; public class IcebergTimestampNsTest { + @Test + public void droppedHistoricalPartitionSourceDoesNotBreakMetadata() { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db")); + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "ts", Types.TimestampType.withoutZone())); + Table table = catalog.createTable(TableIdentifier.of("db", "dropped_source"), schema, + PartitionSpec.builderFor(schema).day("ts").build(), + Collections.singletonMap("format-version", "1")); + table.updateSpec().removeField("ts_day").commit(); + table.updateSchema().deleteColumn("ts").commit(); + table.refresh(); + // V1 retains a void partition field after its source column has been removed. + Assertions.assertFalse(IcebergPartitionUtils.isValidRelatedTable(table)); + } + + @Test + public void negatedPartialConjunctionMustNotPruneMatchingRows() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "ts", Types.DateType.get())); + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db")); + Table table = catalog.createTable(TableIdentifier.of("db", "negated"), schema, + PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "3")); + org.apache.iceberg.TableOperations ops = ((org.apache.iceberg.HasTableOperations) table).operations(); + org.apache.iceberg.TableMetadata base = ops.current(); + ops.commit(base, org.apache.iceberg.TableMetadata.buildFrom(base).setCurrentSchema(new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "ts", Types.TimestampNanoType.withoutZone())), 2).build()); + table.refresh(); + ConnectorComparison ts = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("ts", ConnectorType.of("TIMESTAMP_NS")), + new ConnectorLiteral(ConnectorType.of("STRING"), "2024-01-02")); + ConnectorComparison id = new ConnectorComparison(ConnectorComparison.Operator.GT, + new ConnectorColumnRef("id", ConnectorType.of("INT")), + new ConnectorLiteral(ConnectorType.of("INT"), 0)); + IcebergPredicateConverter converter = IcebergPredicateConverter.forTable( + table, table.schema(), ZoneId.of("UTC"), IcebergPredicateConverter.Mode.SCAN); + ConnectorAnd partial = new ConnectorAnd(java.util.Arrays.asList(ts, id)); + Assertions.assertEquals(1, converter.convert(partial).size()); + // id=1, ts=Jan 3 must survive NOT(ts=Jan 2 AND id>0); NOT(id>0) would lose it. + Assertions.assertTrue(converter.convert(new ConnectorNot(partial)).isEmpty()); + Assertions.assertEquals(1, converter.convert(new ConnectorNot(id)).size()); + } + @Test public void nanosecondCalendarPartitionsDoNotBuildOutOfRangeMvBoundaries() { Schema schema = new Schema(Types.NestedField.optional(1, "ts", Types.TimestampNanoType.withoutZone())); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out index 47e0eeffc34e23..d91b1cbab504c3 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_ns.out @@ -6,6 +6,13 @@ items array Yes true \N detail struct Yes true \N entries map Yes true \N +-- !parquet_tvf_schema -- +id int Yes false \N NONE +ts timestamp_ns Yes false \N NONE +items array Yes false \N NONE +detail struct Yes false \N NONE +entries map Yes false \N NONE + -- !parquet_UTC_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} @@ -190,6 +197,13 @@ items array Yes true \N detail struct Yes true \N entries map Yes true \N +-- !orc_tvf_schema -- +id int Yes false \N NONE +ts timestamp_ns Yes false \N NONE +items array Yes false \N NONE +detail struct Yes false \N NONE +entries map Yes false \N NONE + -- !orc_UTC_values -- 1 2024-01-02 03:04:05.123456789 ["2024-01-02 03:04:05.123456789", null] {"t":"1969-12-31 23:59:58.999999999"} {"x":"1970-01-01 00:00:00.000000001"} 2 2024-01-02 03:04:05.123456788 [] \N {} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy index 9da757c7d13980..85edd1dfed1317 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_ns.groovy @@ -54,6 +54,21 @@ suite("test_iceberg_timestamp_ns", "p0,external,iceberg") { (6, '1677-09-21 00:12:43.145224192', NULL, NULL, NULL), (7, '2262-04-11 23:47:16.854775807', NULL, NULL, NULL)""" "qt_${format}_schema" "DESC ${table}" + // TVF schema discovery must preserve nanos before the V2 scan constructs target columns. + def expectedRows = sql("SELECT * FROM ${table} ORDER BY id") + def tvfRows = [] + def files = sql("SELECT file_path FROM ${table}\$files WHERE content=0") + assertTrue(!files.isEmpty()) + for (def file : files) { + String tvf = """S3("uri"="${file[0]}", "format"="${format}", + "s3.endpoint"="http://${host}:${minioPort}", "s3.access_key"="admin", + "s3.secret_key"="password", "s3.region"="us-east-1", "use_path_style"="true")""" + if (file == files[0]) { + "qt_${format}_tvf_schema" "DESC FUNCTION ${tvf}" + } + tvfRows.addAll(sql("SELECT * FROM ${tvf}")) + } + assertEquals(expectedRows, tvfRows.sort { a, b -> a[0] <=> b[0] }) for (String sessionZone : ["UTC", "Asia/Shanghai"]) { sql "SET time_zone='${sessionZone}'" String tag = "${format}_${sessionZone.replace('/', '_')}" From 85d1412fccf4c085604a751c48e00dcce30e71b7 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 15 Oct 2026 09:58:17 +0800 Subject: [PATCH 13/13] [fix](regression) Align TVF cases with V2 schema inference ### What problem does this PR solve? Related PR: #68934 Problem Summary: TVF schema discovery now honors the V2 logical annotations, but three Parquet suites still expected legacy timestamp and annotated-byte-array results. Two corrupt-file suites also matched legacy reader error wrappers. Regenerate the Parquet goldens from actual execution, pin and restore the group5 display timezone, and assert the concrete invalid-footer errors. ### Release note None ### Check List (For Author) - Test: Reproduce all five non-muted failures locally; run the five complete regression suites against rebuilt FE/BE; 30 Paimon/Iceberg type-mapping unit tests after rebase; FE Checkstyle, clang-format 16 and build hygiene. - Behavior changed: No, align regression expectations with the existing V2 schema behavior. - Does this need documentation: No. --- .../tvf/test_hdfs_parquet_group2.out | 90 +++++++++---------- .../tvf/test_hdfs_parquet_group5.out | 6 +- .../tvf/test_hdfs_parquet_group6.out | 18 ++-- .../test_orc_exception_files.groovy | 15 ++-- .../tvf/test_hdfs_parquet_group2.groovy | 2 + .../tvf/test_hdfs_parquet_group5.groovy | 4 + .../tvf/test_hdfs_parquet_group6.groovy | 2 + .../load_p0/tvf/test_tvf_error_column.groovy | 4 +- 8 files changed, 75 insertions(+), 66 deletions(-) diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out index 902d463fffead5..25d8e521636066 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out @@ -39,14 +39,14 @@ apple_banana_mango9 [{"one":"2 - 0 - 1", "two":"2 - 0 - 2", "three":"2 - 0 - 3"}, {"one":"2 - 1 - 1", "two":"2 - 1 - 2", "three":"2 - 1 - 3"}] -- !test_4 -- -1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 +1001-01-07 17:07:46.123456+08:05:43 1001-01-07 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-08 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-09 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-10 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-11 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-12 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-13 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-14 17:07:46.123456+08:05:43 -- !test_5 -- ["good", "bye"] @@ -92,14 +92,14 @@ apple_banana_mango9 1981-01-10T00:00 20 -- !test_13 -- -1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 +1001-01-07 17:07:46.123456+08:05:43 1001-01-07 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-08 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-09 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-10 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-11 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-12 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-13 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-14 17:07:46.123456+08:05:43 -- !test_14 -- [{"one":"First inner", "two":null, "three":null}, {"one":null, "two":"Second inner", "three":null}, {"one":null, "two":null, "three":"Third inner"}] @@ -171,14 +171,14 @@ apple_banana_mango9 1001-01-07 1001-01-14 -- !test_22 -- -1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 +1001-01-07 17:07:46.123+08:05:43 1001-01-07 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-08 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-09 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-10 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-11 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-12 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-13 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-14 17:07:46.123+08:05:43 -- !test_23 -- 1001-01-07T09:02:03.123456 1001-01-07T09:02:03.123456 @@ -191,16 +191,16 @@ apple_banana_mango9 1001-01-07T09:02:03.123456 1001-01-14T09:02:03.123456 -- !test_24 -- -false 1 2 3 10 1.2 val_1 val_1 0x484541525453 false 1 2 3 10 1.2 val_1 val_1 0x484541525453 ["arr_1", "arr_2", "arr_3"] [1] {1:"val_1", 2:"val_2", 3:"val_3"} {1:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 2:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 3:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}]} -false 3 4 5 30 3.2 val_3 val_3 0x434C554253 \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} -false 5 6 7 50 5.2 val_5 val_5 0x484541525453 false 5 6 7 50 5.2 val_5 val_5 0x484541525453 ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} -false 7 8 9 70 7.2 val_7 val_7 0x434C554253 false 7 8 9 70 7.2 val_7 val_7 0x434C554253 ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} -false 9 10 11 90 9.2 val_9 val_9 0x484541525453 \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} -true 0 1 2 0 0.2 val_0 val_0 0x535041444553 \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} -true 2 3 4 20 2.2 val_2 val_2 0x4449414D4F4E4453 true 2 3 4 20 2.2 val_2 val_2 0x4449414D4F4E4453 ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} -true 4 5 6 40 4.2 val_4 val_4 0x535041444553 true 4 5 6 40 4.2 val_4 val_4 0x535041444553 ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} -true 6 7 8 60 6.2 val_6 val_6 0x4449414D4F4E4453 \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} -true 8 9 10 80 8.2 val_8 val_8 0x535041444553 true 8 9 10 80 8.2 val_8 val_8 0x535041444553 ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} +false 1 2 3 10 1.2 val_1 val_1 HEARTS false 1 2 3 10 1.2 val_1 val_1 HEARTS ["arr_1", "arr_2", "arr_3"] [1] {1:"val_1", 2:"val_2", 3:"val_3"} {1:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 2:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 3:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}]} +false 3 4 5 30 3.2 val_3 val_3 CLUBS \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} +false 5 6 7 50 5.2 val_5 val_5 HEARTS false 5 6 7 50 5.2 val_5 val_5 HEARTS ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} +false 7 8 9 70 7.2 val_7 val_7 CLUBS false 7 8 9 70 7.2 val_7 val_7 CLUBS ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} +false 9 10 11 90 9.2 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} +true 0 1 2 0 0.2 val_0 val_0 SPADES \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} +true 2 3 4 20 2.2 val_2 val_2 DIAMONDS true 2 3 4 20 2.2 val_2 val_2 DIAMONDS ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} +true 4 5 6 40 4.2 val_4 val_4 SPADES true 4 5 6 40 4.2 val_4 val_4 SPADES ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} +true 6 7 8 60 6.2 val_6 val_6 DIAMONDS \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} +true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} -- !test_25 -- {"duration":"111222333444"} @@ -260,9 +260,9 @@ true 8 9 10 80 8.2 val_8 val_8 0x535041444553 true 8 9 10 80 8.2 val_8 val_8 0x5 93.7 -- !test_32 -- -1970-01-01T08:00:00.010 -1970-01-01T08:00:00.010 -1970-01-01T08:00:00.010 +1970-01-01 08:00:00.010+08:00 +1970-01-01 08:00:00.010+08:00 +1970-01-01 08:00:00.010+08:00 -- !test_33 -- 1001-01-07T09:02:03.123456 1001-01-07T09:02:03.123456 @@ -285,14 +285,14 @@ true 8 9 10 80 8.2 val_8 val_8 0x535041444553 true 8 9 10 80 8.2 val_8 val_8 0x5 1001-01-07 1001-01-14 -- !test_35 -- -1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 +1001-01-07 17:07:46.123+08:05:43 1001-01-07 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-08 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-09 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-10 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-11 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-12 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-13 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-14 17:07:46.123+08:05:43 -- !test_36 -- 1001-01-07T09:02:03.123456 1001-01-07T09:02:03.123456 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out index a6f0aae03f9d29..29bac94b8d8980 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out @@ -190,9 +190,9 @@ cta 224.0.90.10 1670922509155338752 15653 EDGX cqs_pillar quote \N \N \N 55 1 R 9624464864560415994 -- !test_23 -- -1 0.1 2017-01-01T07:00 a 2017-01-01T07:00 -2 0.2 2017-01-02T07:00 b 2017-01-02T07:00 -3 0.3 2017-01-03T07:00 c 2017-01-03T07:00 +1 0.1 2017-01-01 07:00:00.000000+08:00 a 2017-01-01 07:00:00.000000+08:00 +2 0.2 2017-01-02 07:00:00.000000+08:00 b 2017-01-02 07:00:00.000000+08:00 +3 0.3 2017-01-03 07:00:00.000000+08:00 c 2017-01-03 07:00:00.000000+08:00 -- !test_24 -- [[["a", "b"], ["c", "d"], ["e"]], [null, ["f"]]] 1 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 309906f05ad500..f0e070e790daf0 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -67,7 +67,7 @@ value 0 value value 0 value value value value value value value value value valu \N \N \N HYUWPBeQIs true \N \N \N \N 5668837.5 -2.1200592435616252e+16 -6097.20 0.00035 -999999999999987628.942299254 \N 2002-11-27T07:59:05.061 l6wBh \N \N \N \N 15002 -304114916502530 \N \N \N 626149930.15083 -342806041448622928.487799977 \N 2009-05-28T13:28:07.366 \N \N true \N 327 \N -6655500289 -64307.047 \N \N \N 559739.491848528 -\N 2016-09-28T15:11:08.215 \N TCo6SPO1zXZ \N 1 \N -2 \N -1.03211568E8 \N -9024.34 23874458.82144 65266925.91513152 +\N 2016-09-28T15:11:08.215 \N TCo6SPO1zXZ \N 1 \N -2 \N -1.03211568E8 \N -9024.34 23874458.82144 65266925.915131517 1991-03-16 2010-11-01T03:14:05.926 LkgyT \N true -1 \N -574 -343357407 \N \N \N \N \N 1993-05-23 2017-03-28T02:26:07.676 i61eSAVRBUP 77e7GSkck5 \N \N 7 \N \N -4198450.5 952.7765389591106 \N \N \N 2005-01-25 \N \N \N -5 \N 1921 \N \N \N \N -999995892.35713 23269406.486042474 @@ -472,10 +472,10 @@ parquet_data_file.parquet 2 -- !test_50 -- \N \N -2023-04-25T20:20:10 2023-04-25T20:20:10 -2023-04-25T20:20:10 2023-04-25T20:20:10 -2023-04-25T20:20:10 2023-04-25T20:20:10 -2023-04-25T20:20:10 2023-04-25T20:20:10 +2023-04-25 20:20:10.000+08:00 2023-04-25 20:20:10.000000+08:00 +2023-04-25 20:20:10.000+08:00 2023-04-25 20:20:10.000000+08:00 +2023-04-25 20:20:10.000+08:00 2023-04-25 20:20:10.000000+08:00 +2023-04-25 20:20:10.000+08:00 2023-04-25 20:20:10.000000+08:00 -- !test_51 -- 20230313151812747 20230313151812747_0_2 uuid:AA0 0e99d67b-6a4c-4aac-b3b0-b74735368a0f-0_0-140-3305_20230313151812747.parquet AA0 10 0 hello [{"array":[10, 20, 30]}, {"array":[40, 50, 60, 70]}] {"key1":[1, 10], "key2":[2, 20], "key3":null} {"a":[10, 20], "b":{"key1":10}, "c":{"a":[10, 20], "b":{"a":10, "b":"world"}}} @@ -657,10 +657,10 @@ hello {"x":"world", "y":"danny"} 5 -- !test_74 -- \N \N \N \N \N -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} +string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} +string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} +string_smith enum_smith 6.7 {"name": "smith"} {"name": "smith"} -- !test_75 -- 20230208203804485 20230208203804485_0_0 AA1 07eeba07-b04d-42b5-9e31-35b1a22ea31d-0_0-89-83_20230208203804485.parquet AA1 20 1 1 2021-01-01 00:00:00.000000+08:00 diff --git a/regression-test/suites/external_table_p0/tvf/orc_format/test_orc_exception_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_format/test_orc_exception_files.groovy index 15b2a61c03bdad..25bad9781f6c7c 100644 --- a/regression-test/suites/external_table_p0/tvf/orc_format/test_orc_exception_files.groovy +++ b/regression-test/suites/external_table_p0/tvf/orc_format/test_orc_exception_files.groovy @@ -35,13 +35,14 @@ suite("test_orc_exception_files", "p0,external") { exception "Footer is corrupt: STRUCT type 0 has 3 subTypes, but has 2 fieldNames" } + // Schema discovery uses V2; assert the corrupt-footer cause, not the reader wrapper. test { uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group3/DwrfStripeCache_BOTH_AllStripes.orc" sql """ select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer" + exception "Failed to parse the footer" } test { @@ -50,7 +51,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } test { @@ -59,7 +60,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } @@ -69,7 +70,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } @@ -79,7 +80,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } test { @@ -88,7 +89,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } test { @@ -97,7 +98,7 @@ suite("test_orc_exception_files", "p0,external") { "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", "format" = "orc"); """ - exception "Init OrcReader failed. reason = Failed to parse the footer from" + exception "Failed to parse the footer from" } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy index f812b2324f7ecf..1c68f1b5982ade 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy @@ -28,6 +28,7 @@ suite("test_hdfs_parquet_group2", "p0,external") { def originalTimeZone = sql "SELECT @@time_zone" try { // Instant-annotated timestamps now retain their timezone-aware type in TVFs. + // V2 schema discovery honors logical UTC annotations even in older-writer fixtures. // Pin the display timezone so the golden results do not depend on the FE default. sql "SET time_zone = 'Asia/Shanghai'" uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/decimal32-written-as-64-bit.snappy.parquet" @@ -198,6 +199,7 @@ suite("test_hdfs_parquet_group2", "p0,external") { "format" = "parquet") limit 10; """ + // V2 schema discovery honors logical annotations instead of exposing raw BYTE_ARRAY. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/parquet-thrift-compat.snappy.parquet" order_qt_test_24 """ select * from HDFS( "uri" = "${uri}", diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy index 8a8a3273d23fd8..37812d9b069b9e 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group5.groovy @@ -25,7 +25,10 @@ suite("test_hdfs_parquet_group5", "p0,external") { String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { + def originalTimeZone = sql "SELECT @@time_zone" try { + // V2 schema discovery retains UTC timestamp annotations; pin their display timezone. + sql "SET time_zone = 'Asia/Shanghai'" uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group5/case_insensitive_column_matching.parquet" order_qt_test_0 """ select * from HDFS( "uri" = "${uri}", @@ -429,6 +432,7 @@ suite("test_hdfs_parquet_group5", "p0,external") { "hadoop.username" = "${hdfsUserName}", "format" = "parquet") limit 10; """ } finally { + sql "SET time_zone = '${originalTimeZone[0][0]}'" } } } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy index b2f0fd778bc288..07a3349d705cd1 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy @@ -28,6 +28,7 @@ suite("test_hdfs_parquet_group6", "p0,external") { def originalTimeZone = sql "SELECT @@time_zone" try { // Instant-annotated timestamps now retain their timezone-aware type in TVFs. + // V2 schema discovery honors logical UTC annotations even in older-writer fixtures. // Pin the display timezone so the golden results do not depend on the FE default. sql "SET time_zone = 'Asia/Shanghai'" uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/upcase_field.parquet" @@ -561,6 +562,7 @@ suite("test_hdfs_parquet_group6", "p0,external") { "format" = "parquet") limit 10; """ + // V2 schema discovery honors logical annotations instead of exposing raw BYTE_ARRAY. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/byte_array.parquet" order_qt_test_74 """ select * from HDFS( "uri" = "${uri}", diff --git a/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy b/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy index 22c2a8a644e593..2fa4cb45410d1d 100644 --- a/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy +++ b/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy @@ -111,7 +111,7 @@ suite("test_tvf_error_column", "p0") { } */ - // Step5: error compression type + // V2 schema discovery must reject CSV bytes as an invalid Parquet footer. test { sql """ INSERT INTO ${tableName} @@ -124,7 +124,7 @@ suite("test_tvf_error_column", "p0") { "region" = "${region}" ); """ - exception "Invalid magic number in parquet file" + exception "Invalid Parquet v2 footer magic" }