Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
d92851e
[fix](paimon) Truncate evolved timestamps to target precision
OIiveirra Sep 17, 2026
5f4e7d2
[fix](paimon) Handle timestamp precision evolution in all readers
OIiveirra Sep 19, 2026
2ca3c3f
[fix](regression) Stabilize Iceberg external fixtures
OIiveirra Sep 19, 2026
7dd90e7
[fix](paimon) Address timestamp precision review findings
OIiveirra Sep 20, 2026
c637de5
[fix](paimon) Scope timestamp precision repair to V2 and JNI
OIiveirra Sep 21, 2026
1f23c15
[refactor](regression) Remove unrelated Iceberg serialization tags
OIiveirra Sep 21, 2026
d679a7f
[fix](paimon) Fix binary scanner materialization
OIiveirra Sep 22, 2026
90577bc
[fix](paimon) Preserve timestamp precision during planning
OIiveirra Sep 23, 2026
4122cce
[test](regression) Add Paimon timestamp repair regressions
OIiveirra Sep 23, 2026
5eea470
[test](regression) Cover native Paimon timestamp predicates
OIiveirra Sep 23, 2026
1e88ec4
[fix](paimon) Preserve safe predicates during timestamp repair
OIiveirra Sep 23, 2026
4eef787
[merge](paimon) Merge latest master into timestamp repair
OIiveirra Oct 11, 2026
55da23d
[refactor](paimon) Remove unrelated changes from timestamp repair PR
OIiveirra Oct 11, 2026
b808b01
[fix](paimon) Preserve timestamp pruning without precision repair
OIiveirra Oct 11, 2026
b0260fe
[fix](paimon) Fix scanner test import ordering
OIiveirra Oct 11, 2026
02b33d5
[test](paimon) Cover compound timestamp predicate repair
OIiveirra Oct 11, 2026
cd19b1c
[fix](regression) Add missing Paimon timestamp fixtures
OIiveirra Oct 12, 2026
f642a70
[fix](regression) Restore Paimon fixture provisioning
OIiveirra Oct 12, 2026
7a66530
[fix](fe) Preserve Paimon timestamp precision across fallback scans
OIiveirra Oct 12, 2026
b374d68
[fix](paimon) Keep ORC routing helper compatible with master tests
OIiveirra Oct 12, 2026
40353c2
[fix](fe) Detect cross-family Paimon timestamp precision narrowing
OIiveirra Oct 13, 2026
8d103cf
[test](regression) Cover cross-family Paimon timestamp narrowing
OIiveirra Oct 13, 2026
ff1efba
[fix](fe) Preserve Paimon timestamp repair across reader paths
OIiveirra Oct 14, 2026
5ed286b
[fix](be) Integrate Paimon precision repair with evolution casts
OIiveirra Oct 15, 2026
1a9eba0
[fix](fe) Reconcile Paimon repair with upstream reader routing
OIiveirra Oct 15, 2026
0b29f2c
[fix](be) Preserve Paimon JNI precision across policy wrappers
OIiveirra Oct 16, 2026
bc433ca
[fix](be) Resolve Paimon policy fields from the selected branch
OIiveirra Oct 17, 2026
a6d3bed
[fix](fe) Preserve branch-local Paimon timestamp repair
OIiveirra Oct 19, 2026
7d129bd
[fix](be) Remove obsolete Variant option from Arrow tests
OIiveirra Oct 19, 2026
9f6d3a5
[fix](be) Bind Paimon timestamp repair reads to the selected branch
OIiveirra Oct 19, 2026
1fff54b
[fix](be) Resolve Paimon chain timestamps against the delta reader
OIiveirra Oct 19, 2026
5b5f93b
[fix](be) Preserve Paimon binlog wrappers during timestamp repair
OIiveirra Oct 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ header:
- "docker/thirdparties/docker-compose/kerberos/paimon_data/**"
- "docker/thirdparties/docker-compose/kerberos/sql/**"
- "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
# Generated Paimon schema and snapshot metadata. These files are copied verbatim into
# MinIO by the Iceberg third-party initializer and cannot carry source comments.
- "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/**/schema/**"
- "docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/**/snapshot/**"
- "docker/thirdparties/docker-compose/postgresql/certs/**"
- "docker/thirdparties/docker-compose/mysql/certs/**"
- "conf/mysql_ssl_default_certificate/*"
Expand Down
28 changes: 27 additions & 1 deletion be/src/format_v2/column_mapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@ namespace doris::format {

namespace {

bool requires_paimon_datetime_precision_conversion(const DataTypePtr& source_type,
const DataTypePtr& target_type) {
const auto source = remove_nullable(source_type);
const auto target = remove_nullable(target_type);
const auto primitive = source->get_primitive_type();
return primitive == target->get_primitive_type() &&
(primitive == TYPE_DATETIMEV2 || primitive == TYPE_TIMESTAMPTZ) &&
Comment thread
OIiveirra marked this conversation as resolved.
Resolved
source->get_scale() > target->get_scale();
}

bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) {
const auto& table_children =
table.identity_children.empty() ? table.children : table.identity_children;
Expand Down Expand Up @@ -1530,6 +1540,11 @@ static bool mapping_can_use_file_column_directly(const ColumnMapping& mapping) {
if (mapping.table_type == nullptr || mapping.file_type == nullptr) {
return false;
}
// A historical .600000 timestamp narrowed to p0 must be materialized through the
// truncating cast; reusing its file column directly would retain the fraction.
if (mapping.truncate_datetimev2_precision) {
return false;
}
const auto table_type = remove_nullable(mapping.table_type);
const auto file_type = remove_nullable(mapping.file_type);
const bool same_timestamptz_with_different_scale =
Expand Down Expand Up @@ -1578,6 +1593,11 @@ static FilterConversionType direct_filter_conversion(const ColumnMapping& mappin
if (type_contains_varbinary(mapping.table_type)) {
return FilterConversionType::FINALIZE_ONLY;
}
// Filtering the historical .600000 value before p0 truncation can reject a row
// that equals the whole-second literal after repair. Keep that predicate above materialization.
if (mapping.truncate_datetimev2_precision) {
return FilterConversionType::FINALIZE_ONLY;
}
const auto table_type = remove_nullable(mapping.table_type);
const auto file_type = remove_nullable(mapping.file_type);
// TIMESTAMPTZ scale mismatch is intentionally materialized as pass-through: a SQL cast rounds
Expand Down Expand Up @@ -2218,7 +2238,8 @@ static void rebuild_projection(ColumnMapping* mapping, LocalIndex block_position
return;
}

auto expr = EvolutionCast::create_shared(mapping->table_type);
auto expr = EvolutionCast::create_shared(mapping->table_type,
mapping->truncate_datetimev2_precision);
expr->add_child(VSlotRef::create_shared(cast_set<int>(block_position.value()),
cast_set<int>(block_position.value()), -1,
mapping->file_type, mapping->file_column_name));
Expand Down Expand Up @@ -2933,6 +2954,11 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c
mapping->projected_file_children = file_field.children;
mapping->timestamp_is_adjusted_to_utc = file_field.timestamp_is_adjusted_to_utc;
mapping->file_type = file_field.type;
const auto file_type = remove_nullable(mapping->file_type);
const auto table_type = remove_nullable(mapping->table_type);
mapping->truncate_datetimev2_precision =
_options.truncate_datetimev2_precision_for_paimon &&
requires_paimon_datetime_precision_conversion(file_type, table_type);
// Access paths are relative to the Variant terminal, so recursive complex mappings must carry
// them instead of leaving them only on the top-level table column.
mapping->variant_access_paths = table_column.variant_access_paths;
Expand Down
6 changes: 6 additions & 0 deletions be/src/format_v2/column_mapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ struct ColumnMapping {
std::vector<ColumnMapping> child_mappings;
// True when file value can be used directly as table value without cast or child remap.
bool is_trivial = false;
// Paimon TIMESTAMP precision narrowing uses truncation instead of the ordinary SQL CAST
// rounding behavior. This is carried recursively so nested timestamp children follow it too.
bool truncate_datetimev2_precision = false;
// How filters referencing this table/global column can be converted below table-reader
// finalize. This is metadata for localize_filters() and future constant-filter evaluation.
FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY;
Expand All @@ -174,6 +177,9 @@ struct TableColumnMapperOptions {
bool enable_row_lineage_virtual_columns = false;
bool enable_iceberg_metadata_virtual_columns = false;
bool enable_paimon_metadata_virtual_columns = false;
// Paimon schema evolution truncates timestamp precision. This must not reuse the SQL CAST
// behavior, which rounds when reducing DATETIMEV2 scale.
bool truncate_datetimev2_precision_for_paimon = false;

std::string debug_string() const;
};
Expand Down
52 changes: 52 additions & 0 deletions be/src/format_v2/expr/evolution_cast.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,63 @@

#include "format_v2/expr/evolution_cast.h"

#include "core/assert_cast.h"
#include "core/column/column_nullable.h"
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_timestamptz.h"
#include "exprs/vexpr_context.h"

namespace doris::format {

Status EvolutionCast::execute_column_impl(VExprContext* context, const Block* block,
const Selector* selector, size_t count,
ColumnPtr& result_column) const {
const auto source = remove_nullable(_children[0]->execute_type(block));
const auto target = remove_nullable(_data_type);
const auto primitive = source->get_primitive_type();
// Paimon evolution truncates 01.600000 at p0 to 01; ordinary CAST rounds it to 02.
// Keep this conversion out of SQL casts and other table formats' evolution paths.
if (!_truncate_datetimev2_precision || primitive != target->get_primitive_type() ||
(primitive != TYPE_DATETIMEV2 && primitive != TYPE_TIMESTAMPTZ) ||
source->get_scale() <= target->get_scale()) {
return Cast::execute_column_impl(context, block, selector, count, result_column);
}

ColumnPtr input;
RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, input));
auto result = _data_type->create_column();
if (is_column_nullable(*result) && !is_column_nullable(*input)) {
auto& nullable_result = assert_cast<ColumnNullable&>(*result);
nullable_result.get_nested_column().insert_range_from(*input, 0, count);
nullable_result.get_null_map_column().insert_many_vals(0, count);
} else {
result->insert_range_from(*input, 0, count);
}
IColumn* nested_column = result.get();
if (is_column_nullable(*nested_column)) {
nested_column = &assert_cast<ColumnNullable&>(*nested_column).get_nested_column();
}
uint32_t divisor = 1;
for (uint32_t i = target->get_scale(); i < 6; ++i) {
divisor *= 10;
}
if (primitive == TYPE_DATETIMEV2) {
auto& data = assert_cast<ColumnDateTimeV2&>(*nested_column).get_data();
for (auto& value : data) {
value.unchecked_set_time_unit<TimeUnit::MICROSECOND>(value.microsecond() / divisor *
divisor);
}
} else {
auto& data = assert_cast<ColumnTimeStampTz&>(*nested_column).get_data();
for (auto& value : data) {
value.set_microsecond(value.microsecond() / divisor * divisor);
}
}
result_column = std::move(result);
return Status::OK();
}

Status EvolutionCast::prepare(RuntimeState* state, const RowDescriptor& desc,
VExprContext* context) {
RETURN_IF_ERROR(Cast::prepare(state, desc, context));
Expand Down
10 changes: 8 additions & 2 deletions be/src/format_v2/expr/evolution_cast.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,21 @@ class EvolutionCast final : public Cast {
ENABLE_FACTORY_CREATOR(EvolutionCast);

public:
explicit EvolutionCast(const DataTypePtr& type) : Cast(type) {}
explicit EvolutionCast(const DataTypePtr& type, bool truncate_datetimev2_precision = false)
: Cast(type), _truncate_datetimev2_precision(truncate_datetimev2_precision) {}

Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override;
Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector,
size_t count, ColumnPtr& result_column) const override;
std::string debug_string() const override;
Status clone_node(VExprSPtr* cloned_expr) const override {
DORIS_CHECK(cloned_expr != nullptr);
*cloned_expr = EvolutionCast::create_shared(_data_type);
*cloned_expr = EvolutionCast::create_shared(_data_type, _truncate_datetimev2_precision);
return Status::OK();
}

private:
bool _truncate_datetimev2_precision = false;
};

} // namespace doris::format
1 change: 1 addition & 0 deletions be/src/format_v2/table/paimon_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class PaimonReader final : public format::TableReader {
protected:
void configure_mapper_options(format::TableColumnMapperOptions* options) const override {
options->enable_paimon_metadata_virtual_columns = true;
options->truncate_datetimev2_precision_for_paimon = true;
}
format::TableColumnMappingMode mapping_mode() const override;
Status annotate_file_schema(std::vector<format::ColumnDefinition>* file_schema) override;
Expand Down
12 changes: 7 additions & 5 deletions be/src/format_v2/table_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -1333,8 +1333,8 @@ class TableReader {
}

Status _cast_column_to_type(ColumnPtr* column, const DataTypePtr& file_type,
const DataTypePtr& table_type,
const std::string& column_name) const {
const DataTypePtr& table_type, const std::string& column_name,
bool truncate_datetimev2_precision = false) const {
DORIS_CHECK(column != nullptr);
DORIS_CHECK(column->get() != nullptr);
DORIS_CHECK(file_type != nullptr);
Expand Down Expand Up @@ -1362,7 +1362,7 @@ class TableReader {
Block cast_block;
cast_block.insert({*column, input_type, column_name});
auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name);
auto cast_expr = EvolutionCast::create_shared(cast_type);
auto cast_expr = EvolutionCast::create_shared(cast_type, truncate_datetimev2_precision);
cast_expr->add_child(std::move(slot_ref));
auto cast_ctx = VExprContext::create_shared(std::move(cast_expr));
RowDescriptor row_desc;
Expand Down Expand Up @@ -1416,7 +1416,8 @@ class TableReader {
// _align_column_nullability() reject an actual NULL before removing the wrapper.
ColumnPtr result_column = source.column;
RETURN_IF_ERROR(_cast_column_to_type(&result_column, slot->data_type(), mapping.table_type,
mapping.file_column_name));
mapping.file_column_name,
mapping.truncate_datetimev2_precision));
RETURN_IF_ERROR(_align_column_nullability(&result_column, mapping.table_type));
*column = _detach_column(std::move(result_column));
*handled = true;
Expand All @@ -1436,7 +1437,8 @@ class TableReader {
nullable_parent_null_map));
} else {
RETURN_IF_ERROR(_cast_column_to_type(column, mapping.file_type, mapping.table_type,
mapping.file_column_name));
mapping.file_column_name,
mapping.truncate_datetimev2_precision));
}
}
RETURN_IF_ERROR(
Expand Down
10 changes: 4 additions & 6 deletions be/test/core/data_type_serde/data_type_serde_arrow_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -957,9 +957,8 @@ TEST(DataTypeSerDeArrowTest, PlainArrowConverterDoesNotInferIcebergUuid) {

TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) {
auto make_variant_column = []() {
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
.throw_on_invalid_json = true,
.check_duplicate_json_path = false});
JsonStringToVariantEncoder encoder(
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json :
{std::string_view {R"({"name":"doris","n":1})"}, std::string_view {"null"},
std::string_view {R"([1,true,"x"])"}}) {
Expand Down Expand Up @@ -1054,9 +1053,8 @@ TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) {

TEST(DataTypeSerDeArrowTest, NestedIcebergVariantExtensionsAndParquetSchema) {
auto make_variant_column = [](std::initializer_list<std::string_view> json_values) {
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
.throw_on_invalid_json = true,
.check_duplicate_json_path = false});
JsonStringToVariantEncoder encoder(
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
for (std::string_view json : json_values) {
encoder.add_json({json.data(), json.size()});
}
Expand Down
103 changes: 103 additions & 0 deletions be/test/format_v2/column_mapper_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "core/column/column_struct.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_decimal.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_nullable.h"
Expand Down Expand Up @@ -3647,6 +3648,108 @@ TEST_F(ColumnMapperCastTest, ColumnMapperBuildsCastProjectionForTypeMismatch) {
mapping.projection->close();
}

TEST_F(ColumnMapperCastTest, PaimonTimestampPrecisionNarrowingTruncatesWithoutRounding) {
const auto file_type = std::make_shared<DataTypeDateTimeV2>(6);
const auto table_type = std::make_shared<DataTypeDateTimeV2>(0);
const std::vector<ColumnDefinition> table_schema = {name_col("ts", table_type)};
const std::vector<ColumnDefinition> file_schema = {name_col("ts", file_type, 0)};
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME,
.truncate_datetimev2_precision_for_paimon = true});
ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok());
ASSERT_EQ(mapper.mappings().size(), 1);
EXPECT_TRUE(mapper.mappings()[0].truncate_datetimev2_precision);
EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY);

FileScanRequest file_request;
ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok());
const auto& mapping = mapper.mappings()[0];
ASSERT_NE(mapping.projection, nullptr);

auto source = ColumnDateTimeV2::create();
DateV2Value<DateTimeV2ValueType> value;
value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000);
source->get_data().push_back(value);
Block block;
block.insert({std::move(source), file_type, "ts"});
int result_column_id = -1;
ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok());

const auto& result =
assert_cast<const ColumnDateTimeV2&>(*block.get_by_position(result_column_id).column);
ASSERT_EQ(result.size(), 1);
EXPECT_EQ(result.get_data()[0].second(), 1);
EXPECT_EQ(result.get_data()[0].microsecond(), 0);
mapping.projection->close();
}

TEST_F(ColumnMapperCastTest, PaimonTimestampTzPrecisionNarrowingTruncatesWithoutRounding) {
const auto file_type = timestamptz(6);
const auto table_type = timestamptz(0);
const std::vector<ColumnDefinition> table_schema = {name_col("ts", table_type)};
const std::vector<ColumnDefinition> file_schema = {name_col("ts", file_type, 0)};
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME,
.truncate_datetimev2_precision_for_paimon = true});
ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok());
ASSERT_EQ(mapper.mappings().size(), 1);
EXPECT_TRUE(mapper.mappings()[0].truncate_datetimev2_precision);
EXPECT_FALSE(mapper.mappings()[0].is_trivial);
EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::FINALIZE_ONLY);

FileScanRequest file_request;
ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok());
const auto& mapping = mapper.mappings()[0];
ASSERT_NE(mapping.projection, nullptr);

auto source = ColumnTimeStampTz::create();
TimestampTzValue value;
value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000);
source->get_data().push_back(value);
Block block;
block.insert({std::move(source), file_type, "ts"});
int result_column_id = -1;
ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok());

const auto& result =
assert_cast<const ColumnTimeStampTz&>(*block.get_by_position(result_column_id).column);
ASSERT_EQ(result.size(), 1);
EXPECT_EQ(result.get_data()[0].second(), 1);
EXPECT_EQ(result.get_data()[0].microsecond(), 0);
mapping.projection->close();
}

TEST_F(ColumnMapperCastTest, PaimonTimestampPrecisionNarrowingWrapsRequiredFileColumn) {
const auto file_type = std::make_shared<DataTypeDateTimeV2>(6);
const auto table_type = make_nullable(std::make_shared<DataTypeDateTimeV2>(0));
const std::vector<ColumnDefinition> table_schema = {name_col("ts", table_type)};
const std::vector<ColumnDefinition> file_schema = {name_col("ts", file_type, 0)};
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME,
.truncate_datetimev2_precision_for_paimon = true});
ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok());

FileScanRequest file_request;
ASSERT_TRUE(mapper.create_scan_request({}, table_schema, &file_request).ok());
const auto& mapping = mapper.mappings()[0];
ASSERT_NE(mapping.projection, nullptr);

auto source = ColumnDateTimeV2::create();
DateV2Value<DateTimeV2ValueType> value;
value.unchecked_set_time(2025, 1, 1, 0, 0, 1, 600000);
source->get_data().push_back(value);
Block block;
block.insert({std::move(source), file_type, "ts"});
int result_column_id = -1;
ASSERT_TRUE(prepare_open_execute(mapping.projection.get(), &block, &result_column_id).ok());

const auto& result =
assert_cast<const ColumnNullable&>(*block.get_by_position(result_column_id).column);
ASSERT_EQ(result.size(), 1);
EXPECT_FALSE(result.is_null_at(0));
const auto& nested = assert_cast<const ColumnDateTimeV2&>(result.get_nested_column());
EXPECT_EQ(nested.get_data()[0].second(), 1);
EXPECT_EQ(nested.get_data()[0].microsecond(), 0);
mapping.projection->close();
}

// Scenario: equivalent table/file types keep the mapping trivial and avoid unnecessary projection casts.
TEST_F(ColumnMapperCastTest, ColumnMapperTreatsEquivalentTypesAsTrivial) {
TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME});
Expand Down
Loading
Loading