diff --git a/be/src/exec/scan/file_scan_range_utils.h b/be/src/exec/scan/file_scan_range_utils.h index a9bf6cbcedfdeb..a6cc66dae6d262 100644 --- a/be/src/exec/scan/file_scan_range_utils.h +++ b/be/src/exec/scan/file_scan_range_utils.h @@ -91,10 +91,11 @@ inline Status validate_file_scan_support(const TFileScanRangeParams& params, } inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) { - // Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a - // missing data or delete file would silently return an incomplete or incorrect snapshot. - return ignore_not_found && !(range.__isset.table_format_params && - range.table_format_params.table_format_type == "iceberg"); + // Iceberg and Delta ranges belong to a committed snapshot. Skipping a missing data or + // delete file would silently return a partial snapshot. + const auto& table = range.table_format_params.table_format_type; + return ignore_not_found && + !(range.__isset.table_format_params && (table == "iceberg" || table == "deltalake")); } } // namespace doris diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 86d261b3397cb0..879f5cfcfb18cc 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -60,6 +60,7 @@ #include "format_v2/jni/max_compute_jni_reader.h" #include "format_v2/jni/trino_connector_jni_reader.h" #include "format_v2/table/adbc_reader.h" +#include "format_v2/table/deltalake_reader.h" #include "format_v2/table/es_reader.h" #include "format_v2/table/fluss_hybrid_reader.h" #include "format_v2/table/hive_reader.h" @@ -668,6 +669,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(); } else if (table_format == "hudi") { *reader = std::make_unique(); + } else if (table_format == "deltalake") { + *reader = std::make_unique(); } else if (table_format == "fluss") { *reader = std::make_unique(); } else if (table_format == "jdbc") { diff --git a/be/src/format/table/deletion_vector_reader.cpp b/be/src/format/table/deletion_vector_reader.cpp index 946e81fa2e6348..758616c8bf5f38 100644 --- a/be/src/format/table/deletion_vector_reader.cpp +++ b/be/src/format/table/deletion_vector_reader.cpp @@ -103,6 +103,16 @@ Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read); } +Status validate_delta_deletion_vector_read_range(int64_t offset, int32_t size_in_bytes, + size_t& bytes_read) { + // Delta and Iceberg share the portable bitmap framing. Delta's descriptor omits the + // four-byte length and four-byte CRC; promote before adding them to avoid i32 overflow. + return validate_deletion_vector_read_range( + "Delta deletion vector", offset, static_cast(size_in_bytes) + 8, + ICEBERG_DELETION_VECTOR_MIN_BYTES, MAX_ICEBERG_DELETION_VECTOR_BYTES, + DeletionVectorSizeLimitStatus::NOT_SUPPORTED, bytes_read); +} + DeletionVectorReader::~DeletionVectorReader() { // The file reader may retain the child IOContext. Destroy it before merging and before the // child statistics storage goes away. diff --git a/be/src/format/table/deletion_vector_reader.h b/be/src/format/table/deletion_vector_reader.h index c2ed07a242aa42..a02e9e48e7d66a 100644 --- a/be/src/format/table/deletion_vector_reader.h +++ b/be/src/format/table/deletion_vector_reader.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -44,10 +45,14 @@ Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, size_t& bytes_read); +Status validate_delta_deletion_vector_read_range(int64_t offset, int32_t size_in_bytes, + size_t& bytes_read); + struct DeleteFileDesc { enum class Format { PAIMON, ICEBERG, + DELTA, }; std::string key = ""; @@ -58,6 +63,8 @@ struct DeleteFileDesc { int64_t file_size = -1; int64_t modification_time = 0; Format format = Format::PAIMON; + std::optional inline_data = std::nullopt; + std::optional expected_cardinality = std::nullopt; }; class DeletionVectorReader { diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 4c5e735444fe97..5c3b741609c4b7 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -52,6 +52,24 @@ namespace { constexpr const char* ICEBERG_FILE_PATH = "file_path"; constexpr const char* ICEBERG_ROW_POS = "pos"; constexpr size_t ICEBERG_DELETION_VECTOR_MIN_BYTES = 12; +constexpr char PORTABLE_DELETION_VECTOR_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; + +Status validate_portable_deletion_vector_magic(const char* buf) { + if (memcmp(buf, PORTABLE_DELETION_VECTOR_MAGIC, sizeof(PORTABLE_DELETION_VECTOR_MAGIC)) != 0) { + return Status::DataQualityError("Deletion vector magic number mismatch"); + } + return Status::OK(); +} + +Status decode_portable_deletion_vector_bitmap(const char* buf, size_t size, + DeletionVector* rows_to_delete) { + try { + *rows_to_delete |= roaring::Roaring64Map::readSafe(buf, size); + } catch (const std::runtime_error& e) { + return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); + } + return Status::OK(); +} Status validate_position_delete_file_format(const TIcebergDeleteFileDesc& delete_file, TFileFormatType::type* file_format) { @@ -144,10 +162,7 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, static_cast(total_length) + 8, buffer_size); } - constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; - if (memcmp(buf + sizeof(total_length), MAGIC_NUMBER, 4) != 0) { - return Status::DataQualityError("Deletion vector magic number mismatch"); - } + RETURN_IF_ERROR(validate_portable_deletion_vector_magic(buf + sizeof(total_length))); const uint32_t expected_crc = BigEndian::Load32(buf + sizeof(total_length) + total_length); const uint32_t actual_crc = @@ -157,13 +172,8 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, expected_crc, actual_crc); } - try { - *rows_to_delete |= roaring::Roaring64Map::readSafe( - buf + 8, buffer_size - ICEBERG_DELETION_VECTOR_MIN_BYTES); - } catch (const std::runtime_error& e) { - return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); - } - return Status::OK(); + return decode_portable_deletion_vector_bitmap( + buf + 8, buffer_size - ICEBERG_DELETION_VECTOR_MIN_BYTES, rows_to_delete); } } // namespace @@ -324,4 +334,16 @@ Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size return decode_deletion_vector_buffer(buf, buffer_size, rows_to_delete); } +Status decode_portable_deletion_vector_payload(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete) { + DORIS_CHECK(buf != nullptr); + DORIS_CHECK(rows_to_delete != nullptr); + // Even an empty portable Roaring64Map has an eight-byte map count after the magic. + if (buffer_size < 12) { + return Status::DataQualityError("Deletion vector payload size too small: {}", buffer_size); + } + RETURN_IF_ERROR(validate_portable_deletion_vector_magic(buf)); + return decode_portable_deletion_vector_bitmap(buf + 4, buffer_size - 4, rows_to_delete); +} + } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.h b/be/src/format/table/iceberg_delete_file_reader_helper.h index d43443ecd8e1c9..a186399f2f1d7a 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.h +++ b/be/src/format/table/iceberg_delete_file_reader_helper.h @@ -80,6 +80,11 @@ Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, DeletionVector* rows_to_delete); +// Portable DV magic and bitmap, without the on-disk length prefix and CRC, as carried by +// inline Delta DVs. On-disk Iceberg/Delta DVs use decode_iceberg_deletion_vector_buffer. +Status decode_portable_deletion_vector_payload(const char* buf, size_t buffer_size, + DeletionVector* rows_to_delete); + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor); diff --git a/be/src/format_v2/table/deltalake_reader.cpp b/be/src/format_v2/table/deltalake_reader.cpp new file mode 100644 index 00000000000000..4eb3b80790f7f1 --- /dev/null +++ b/be/src/format_v2/table/deltalake_reader.cpp @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/deltalake_reader.h" + +#include "format/table/deletion_vector_reader.h" +#include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_reader.h" +#include "format_v2/table/schema_history_util.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_state.h" + +namespace doris::format::deltalake { + +Status DeltaReader::prepare_split(const SplitReadOptions& options) { + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& params = options.current_range.table_format_params; + if (params.__isset.deltalake_params && params.deltalake_params.__isset.schema_id) { + _split_schema_id = params.deltalake_params.schema_id; + DORIS_CHECK(can_map_by_history_schema(_scan_params, _split_schema_id)); + } + } + RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (current_split_pruned()) { + return Status::OK(); + } + // Delta commits replace data files instead of mutating them. A file may acquire a new DV, + // but its Parquet bytes (and therefore its page-cache identity) remain unchanged. + mark_current_data_file_immutable(); + return Status::OK(); +} + +TableColumnMappingMode DeltaReader::mapping_mode() const { + return can_map_by_history_schema(_scan_params, _split_schema_id) + ? TableColumnMappingMode::BY_FIELD_ID + : TableColumnMappingMode::BY_NAME; +} + +Status DeltaReader::create_file_reader(std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(_format == FileFormat::PARQUET); + DORIS_CHECK(_runtime_state != nullptr); + const bool mapping_timestamp_tz = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_timestamp_tz && + _scan_params->enable_mapping_timestamp_tz; + const bool mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; + // Spark's legacy INT96 timestamps are instants. Use the query timezone, independently + // of Hive's INT96 compatibility setting; timestamp_ntz keeps its Parquet wall-clock flag. + *reader = std::make_unique( + _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, + _global_rowid_context, mapping_timestamp_tz, mapping_varbinary, + _runtime_state->timezone(), false, _parquet_profile_cache); + return Status::OK(); +} + +Status DeltaReader::annotate_file_schema(std::vector* file_schema) { + DORIS_CHECK(file_schema != nullptr); + if (mapping_mode() == TableColumnMappingMode::BY_FIELD_ID) { + return annotate_file_schema_from_history(_scan_params, _split_schema_id, file_schema); + } + return Status::OK(); +} + +Status DeltaReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, + DeleteFileDesc* desc, bool* has_delete_file) { + DORIS_CHECK(desc != nullptr); + DORIS_CHECK(has_delete_file != nullptr); + *has_delete_file = false; + const auto& params = t_desc.deltalake_params; + if (!params.__isset.deletion_vector) { + return Status::OK(); + } + const auto& dv = params.deletion_vector; + DORIS_CHECK(dv.__isset.size_in_bytes); + DORIS_CHECK(dv.__isset.cardinality); + if (dv.cardinality < 0) { + return Status::DataQualityError("Delta deletion vector cardinality is negative: {}", + dv.cardinality); + } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_delta_deletion_vector_read_range(dv.__isset.offset ? dv.offset : 0, + dv.size_in_bytes, bytes_read)); + desc->format = DeleteFileDesc::Format::DELTA; + desc->expected_cardinality = static_cast(dv.cardinality); + if (dv.__isset.inline_data) { + DORIS_CHECK(!dv.__isset.path); + if (dv.inline_data.size() != static_cast(dv.size_in_bytes)) { + return Status::DataQualityError( + "Inline Delta deletion vector size mismatch, expected: {}, actual: {}", + dv.size_in_bytes, dv.inline_data.size()); + } + desc->inline_data = dv.inline_data; + desc->size = dv.size_in_bytes; + // The exact bytes avoid hash collisions and distinguish different inline DVs without + // performing I/O. Binary strings, including embedded NULs, are valid KV-cache keys. + desc->key = "deltalake_dv_inline_" + dv.inline_data; + } else { + DORIS_CHECK(dv.__isset.path); + desc->path = dv.path; + desc->start_offset = dv.__isset.offset ? dv.offset : 0; + desc->size = static_cast(bytes_read); + desc->key = + fmt::format("deltalake_dv_{}#{}#{}", desc->path, desc->start_offset, desc->size); + } + *has_delete_file = true; + return Status::OK(); +} + +} // namespace doris::format::deltalake diff --git a/be/src/format_v2/table/deltalake_reader.h b/be/src/format_v2/table/deltalake_reader.h new file mode 100644 index 00000000000000..487fa4fcba9eda --- /dev/null +++ b/be/src/format_v2/table/deltalake_reader.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "format_v2/table_reader.h" + +namespace doris::format::deltalake { + +class DeltaReader final : public TableReader { +public: + ~DeltaReader() override = default; + Status prepare_split(const SplitReadOptions& options) override; + +protected: + Status create_file_reader(std::unique_ptr* reader) override; + TableColumnMappingMode mapping_mode() const override; + Status annotate_file_schema(std::vector* file_schema) override; + Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, + bool* has_delete_file) override; + +private: + int64_t _split_schema_id = -1; +}; + +} // namespace doris::format::deltalake diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 0c25797633e58e..6934d0ef9aef87 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -587,14 +587,19 @@ Status build_table_filters_from_conjunct(const VExprContextSPtr& conjunct, Runti return Status::OK(); } -Status parse_deletion_vector(const char* buf, size_t buffer_size, DeleteFileDesc::Format format, +Status parse_deletion_vector(const char* buf, size_t buffer_size, const DeleteFileDesc& desc, DeletionVector* deletion_vector) { DORIS_CHECK(buf != nullptr); DORIS_CHECK(deletion_vector != nullptr); - DORIS_CHECK(format == DeleteFileDesc::Format::PAIMON || - format == DeleteFileDesc::Format::ICEBERG); + DORIS_CHECK(desc.format == DeleteFileDesc::Format::PAIMON || + desc.format == DeleteFileDesc::Format::ICEBERG || + desc.format == DeleteFileDesc::Format::DELTA); - if (format == DeleteFileDesc::Format::PAIMON) { + if (desc.inline_data.has_value()) { + DORIS_CHECK(desc.format == DeleteFileDesc::Format::DELTA); + return decode_portable_deletion_vector_payload(buf, buffer_size, deletion_vector); + } + if (desc.format == DeleteFileDesc::Format::PAIMON) { RETURN_IF_ERROR(decode_paimon_deletion_vector_buffer(buf, buffer_size, deletion_vector)); return Status::OK(); } @@ -1568,41 +1573,46 @@ Status TableReader::_parse_delete_predicates(const SplitReadOptions& options) { [&]() -> DeletionVector* { auto deletion_vector = std::make_unique(); - DeletionVectorReader dv_reader(_runtime_state, _scanner_profile, *_scan_params, - desc, _io_ctx.get()); - create_status = dv_reader.open(); - if (!create_status.ok()) [[unlikely]] { - return nullptr; - } - - size_t bytes_read = desc.size; - std::vector buffer(bytes_read); - DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", { + const size_t bytes_read = desc.size; + std::vector buffer; + const char* buf = nullptr; + if (desc.inline_data.has_value()) { + buf = desc.inline_data->data(); + } else { + DeletionVectorReader dv_reader(_runtime_state, _scanner_profile, + *_scan_params, desc, _io_ctx.get()); + create_status = dv_reader.open(); + if (!create_status.ok()) [[unlikely]] { + return nullptr; + } + buffer.resize(bytes_read); + DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", { + create_status = Status::IOError( + "injected format v2 deletion vector read failure"); + return nullptr; + }); + DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.should_stop", { + create_status = Status::EndOfFile("stop read."); + return nullptr; + }); create_status = - Status::IOError("injected format v2 deletion vector read failure"); - return nullptr; - }); - DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.should_stop", { - create_status = Status::EndOfFile("stop read."); - return nullptr; - }); - create_status = - dv_reader.read_at(desc.start_offset, {buffer.data(), bytes_read}); - const auto& file_cache_stats = dv_reader.file_cache_statistics(); - COUNTER_UPDATE(_profile.dv_file_cache_hit_count, - file_cache_stats.num_local_io_total); - COUNTER_UPDATE(_profile.dv_file_cache_miss_count, - file_cache_stats.num_remote_io_total); - COUNTER_UPDATE(_profile.dv_file_cache_peer_read_count, - file_cache_stats.num_peer_io_total); - if (!create_status.ok()) [[unlikely]] { - return nullptr; + dv_reader.read_at(desc.start_offset, {buffer.data(), bytes_read}); + const auto& file_cache_stats = dv_reader.file_cache_statistics(); + COUNTER_UPDATE(_profile.dv_file_cache_hit_count, + file_cache_stats.num_local_io_total); + COUNTER_UPDATE(_profile.dv_file_cache_miss_count, + file_cache_stats.num_remote_io_total); + COUNTER_UPDATE(_profile.dv_file_cache_peer_read_count, + file_cache_stats.num_peer_io_total); + if (!create_status.ok()) [[unlikely]] { + return nullptr; + } + buf = buffer.data(); } - const char* buf = buffer.data(); SCOPED_TIMER(_profile.parse_delete_file_time); - create_status = parse_deletion_vector(buf, bytes_read, desc.format, - deletion_vector.get()); + create_status = + parse_deletion_vector(buf, bytes_read, desc, deletion_vector.get()); if (!create_status.ok()) [[unlikely]] { return nullptr; } @@ -1611,6 +1621,14 @@ Status TableReader::_parse_delete_predicates(const SplitReadOptions& options) { }, &decoded_cache_hit); RETURN_IF_ERROR(create_status); + // Check on cache hits as well: a corrupt log entry can reuse a valid bitmap while + // advertising a different cardinality, which would make metadata count disagree. + if (desc.expected_cardinality.has_value() && + _deletion_vector->cardinality() != *desc.expected_cardinality) { + return Status::DataQualityError( + "Deletion vector cardinality mismatch, expected: {}, actual: {}", + *desc.expected_cardinality, _deletion_vector->cardinality()); + } COUNTER_UPDATE(decoded_cache_hit ? _profile.decoded_dv_cache_hit_count : _profile.decoded_dv_cache_miss_count, 1); diff --git a/be/test/exec/operator/file_scan_operator_deltalake_test.cpp b/be/test/exec/operator/file_scan_operator_deltalake_test.cpp new file mode 100644 index 00000000000000..8141485b9008d5 --- /dev/null +++ b/be/test/exec/operator/file_scan_operator_deltalake_test.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "exec/scan/file_scan_range_utils.h" +#include "exec/scan/file_scanner_v2.h" +#include "format_v2/table/deltalake_reader.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_state.h" + +namespace doris { + +TEST(FileScanOperatorDeltaLakeTest, DispatchesNativeReaderRegardlessOfDeprecatedScannerOption) { + for (int option = 0; option < 3; ++option) { + TQueryOptions options; + if (option != 0) { + options.__set_enable_file_scanner_v2(option == 2); + } + RuntimeState state(options, TQueryGlobals()); + RuntimeProfile profile("delta_scan"); + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + FileScannerV2 scanner(&state, &profile, nullptr); + scanner._params = ¶ms; + TTableFormatFileDesc table; + table.__set_table_format_type("deltalake"); + TFileRangeDesc range; + range.__set_table_format_params(table); + std::unique_ptr reader; + ASSERT_TRUE(validate_file_scan_support(params, &range).ok()); + ASSERT_TRUE(scanner._create_table_reader_for_format(range, &reader).ok()); + EXPECT_NE(dynamic_cast(reader.get()), nullptr); + } +} + +TEST(FileScanOperatorDeltaLakeTest, MissingSnapshotFilesFailEvenWhenDirectoryReadsIgnoreNotFound) { + TTableFormatFileDesc table; + table.__set_table_format_type("deltalake"); + TFileRangeDesc range; + range.__set_table_format_params(table); + EXPECT_FALSE(can_ignore_not_found_file(range, true)); + EXPECT_FALSE(can_ignore_not_found_file(range, false)); +} + +} // namespace doris diff --git a/be/test/format_v2/table/deltalake_reader_test.cpp b/be/test/format_v2/table/deltalake_reader_test.cpp new file mode 100644 index 00000000000000..191f89eca48c37 --- /dev/null +++ b/be/test/format_v2/table/deltalake_reader_test.cpp @@ -0,0 +1,752 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/deltalake_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "exec/common/endian.h" +#include "format/format_common.h" +#include "format/table/deletion_vector_reader.h" +#include "format/table/iceberg_delete_file_reader_helper.h" +#include "format_v2/column_mapper.h" +#include "gen_cpp/ExternalTableSchema_types.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "util/debug_points.h" +#include "util/timezone_utils.h" + +namespace doris::format { +namespace { + +constexpr const char* DATA_FILE = + "part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet"; +constexpr const char* DV_FILES[] = {"deletion_vector_37d10da3-70a1-4730-bc58-3b44f9617505.bin", + "deletion_vector_3d8a467a-2fbd-4d35-8e3a-775894a30576.bin", + "deletion_vector_d12e7d16-e46d-48c9-8a71-b222c26dfc3b.bin"}; + +std::string fixture_path(const char* name) { + return (std::filesystem::path(std::getenv("ROOT")) / + "be/test/format_v2/table/test_data/deltalake" / name) + .string(); +} + +std::string read_fixture(const char* name) { + std::ifstream file(fixture_path(name), std::ios::binary); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); +} + +TDeltaLakeDeletionVectorDesc disk_dv(size_t index) { + TDeltaLakeDeletionVectorDesc dv; + dv.__set_path(fixture_path(DV_FILES[index])); + dv.__set_offset(1); + dv.__set_size_in_bytes(34 + static_cast(index) * 2); + dv.__set_cardinality(static_cast(index) + 1); + return dv; +} + +TDeltaLakeDeletionVectorDesc inline_dv(size_t index) { + auto dv = disk_dv(index); + dv.__isset.path = false; + dv.__isset.offset = false; + const auto file = read_fixture(DV_FILES[index]); + dv.__set_inline_data(file.substr(5, dv.size_in_bytes)); + return dv; +} + +TTableFormatFileDesc table_params(std::optional dv = std::nullopt, + std::optional schema_id = std::nullopt) { + TDeltaLakeFileDesc params; + if (dv.has_value()) { + params.__set_deletion_vector(*dv); + } + if (schema_id.has_value()) { + params.__set_schema_id(*schema_id); + } + TTableFormatFileDesc result; + result.__set_table_format_type("deltalake"); + result.__set_deltalake_params(params); + return result; +} + +ColumnDefinition table_column(int id, const std::string& name) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = name; + column.type = make_nullable(std::make_shared()); + return column; +} + +ColumnDefinition typed_column(int id, const std::string& name, DataTypePtr type) { + auto column = table_column(id, name); + column.type = make_nullable(std::move(type)); + return column; +} + +schema::external::TFieldPtr schema_field(const std::string& name, int id) { + schema::external::TFieldPtr result; + result.field_ptr = std::make_shared(); + result.__isset.field_ptr = true; + result.field_ptr->__set_name(name); + result.field_ptr->__set_id(id); + return result; +} + +schema::external::TSchema schema(int64_t id, std::vector fields) { + schema::external::TStructField root; + root.__set_fields(std::move(fields)); + schema::external::TSchema result; + result.__set_schema_id(id); + result.__set_root_field(root); + return result; +} + +class FailDeletionVectorIO { +public: + FailDeletionVectorIO() : _previous(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add("TableReader.parse_deletion_vector.io_error"); + } + + ~FailDeletionVectorIO() { + DebugPoints::instance()->remove("TableReader.parse_deletion_vector.io_error"); + config::enable_debug_points = _previous; + } + +private: + bool _previous; +}; + +class DeltaReaderTest : public testing::Test { +protected: + static void SetUpTestSuite() { TimezoneUtils::load_timezones_to_cache(); } + + void SetUp() override { + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &io_stats; + io_ctx->file_cache_stats = &file_cache_stats; + } + + Status init(deltalake::DeltaReader* reader) { + return reader->init({.projected_columns = columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile}); + } + + SplitReadOptions split(const TTableFormatFileDesc& params, const char* file = DATA_FILE) { + SplitReadOptions options; + options.cache = &cache; + options.current_range.__set_path(fixture_path(file)); + options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(fixture_path(file)))); + options.current_range.__set_start_offset(0); + options.current_range.__set_size(options.current_range.file_size); + options.current_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + options.current_range.__set_table_format_params(params); + return options; + } + + void read_strings(deltalake::DeltaReader* reader, std::vector>* rows) { + bool eos = false; + while (!eos) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + auto status = reader->get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + for (size_t row = 0; row < block.rows(); ++row) { + std::vector values; + for (const auto& column : block.get_columns_with_type_and_name()) { + values.push_back(column.type->to_string(*column.column, row)); + } + rows->push_back(std::move(values)); + } + } + } + + void read(deltalake::DeltaReader* reader, std::vector>* ids, + std::optional partition = std::nullopt) { + bool eos = false; + while (!eos) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + auto status = reader->get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + const auto full_column = + block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& column = assert_cast(*full_column); + const auto& values = assert_cast(column.get_nested_column()); + for (size_t row = 0; row < block.rows(); ++row) { + ids->push_back(column.is_null_at(row) + ? std::nullopt + : std::make_optional(values.get_element(row))); + if (partition.has_value()) { + const auto full_partition = + block.get_by_position(1).column->convert_to_full_column_if_const(); + const auto& p = assert_cast(*full_partition); + EXPECT_FALSE(p.is_null_at(row)); + EXPECT_EQ( + assert_cast(p.get_nested_column()).get_element(row), + *partition); + } + } + } + } + + RuntimeProfile profile {"delta_test"}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams scan_params; + io::FileReaderStats io_stats; + io::FileCacheStatistics file_cache_stats; + std::shared_ptr io_ctx; + ShardedKVCache cache {1}; + std::vector columns {table_column(0, "id")}; +}; + +TEST_F(DeltaReaderTest, ReadsSparkDataWithPartitionAndNoDeletionVector) { + columns.push_back(table_column(1, "partition")); + columns.back().is_partition_key = true; + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + auto options = split(table_params()); + options.partition_values.emplace("partition", Field::create_field(42)); + ASSERT_TRUE(reader.prepare_split(options).ok()); + EXPECT_TRUE(reader.TEST_current_data_file_is_immutable()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_NAME); + std::vector> ids; + read(&reader, &ids, 42); + ASSERT_EQ(ids.size(), 50); + for (int64_t i = 0; i < 50; ++i) { + EXPECT_EQ(ids[i], i); + } + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, DiskAndInlineVectorsFilterIdenticallyAndResetBetweenSplits) { + for (bool inlined : {false, true}) { + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + reader.set_batch_size(3); + for (size_t version = 0; version < 3; ++version) { + auto options = split(table_params(inlined ? inline_dv(version) : disk_dv(version))); + ASSERT_TRUE(reader.prepare_split(options).ok()); + std::vector> ids; + read(&reader, &ids); + std::vector> expected; + for (int64_t i = 0; i < 50; ++i) { + if (i > static_cast(version) * 7 || i % 7 != 0) { + expected.emplace_back(i); + } + } + EXPECT_EQ(ids, expected); + } + ASSERT_TRUE(reader.prepare_split(split(table_params())).ok()); + std::vector> ids; + read(&reader, &ids); + EXPECT_EQ(ids.size(), 50); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST_F(DeltaReaderTest, InlineBypassesIOAndChecksCardinalityOnCacheHits) { + FailDeletionVectorIO fail_io; + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + auto dv = inline_dv(2); + auto options = split(table_params(dv)); + ASSERT_TRUE(reader._parse_delete_predicates(options).ok()); + EXPECT_EQ(io_stats.read_calls, 0); + EXPECT_EQ(reader._deletion_vector->cardinality(), 3); + auto* decoded = reader._deletion_vector; + ASSERT_TRUE(reader._parse_delete_predicates(options).ok()); + EXPECT_EQ(decoded, reader._deletion_vector); + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheMissCount")->value(), 1); + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheHitCount")->value(), 1); + dv.__set_cardinality(4); + auto status = reader._parse_delete_predicates(split(table_params(dv))); + EXPECT_TRUE(status.is()) << status; + EXPECT_EQ(io_stats.read_calls, 0); + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, WidensOldSparkFileAndReadsNewSchemaFile) { + const auto int64_type = std::make_shared(); + const auto double_type = std::make_shared(); + columns = {typed_column(0, "byte_long", int64_type), + typed_column(1, "int_long", int64_type), + typed_column(2, "float_double", double_type), + typed_column(3, "byte_double", double_type), + typed_column(4, "short_double", double_type), + typed_column(5, "int_double", double_type), + typed_column(6, "decimal_decimal_same_scale", create_decimal(20, 2, false)), + typed_column(7, "decimal_decimal_greater_scale", create_decimal(20, 5, false)), + typed_column(8, "byte_decimal", create_decimal(11, 1, false)), + typed_column(9, "short_decimal", create_decimal(11, 1, false)), + typed_column(10, "int_decimal", create_decimal(11, 1, false)), + typed_column(11, "long_decimal", create_decimal(21, 1, false)), + typed_column(12, "date_timestamp_ntz", std::make_shared(6))}; + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + const char* files[] = {"part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet", + "part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet"}; + std::vector> rows; + for (const auto* file : files) { + ASSERT_TRUE(reader.prepare_split(split(table_params(), file)).ok()); + read_strings(&reader, &rows); + } + const std::vector> expected { + {"1", "2", "3.4000000953674316", "5", "6", "7", "123.45", "67.89000", "1.0", "2.0", + "3.0", "4.0", "2024-09-09 00:00:00.000000"}, + {"9223372036854775807", "9223372036854775807", "1.234567890123", "1.234567890123", + "1.234567890123", "1.234567890123", "12345678901234.56", "12345678901.23456", "123.4", + "12345.6", "1234567890.1", "123456789012345678.9", "2024-09-09 12:34:56.123456"}}; + EXPECT_EQ(rows, expected); + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, WidensByteAndShortToLargerIntegers) { + const char* file = "part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet"; + const DataTypes targets {std::make_shared(), std::make_shared(), + std::make_shared()}; + for (size_t i = 0; i < targets.size(); ++i) { + columns = {typed_column(0, "byte_long", targets[i])}; + if (i > 0) { + columns.push_back(typed_column(1, "short_double", targets[i])); + } + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + ASSERT_TRUE(reader.prepare_split(split(table_params(), file)).ok()); + std::vector> rows; + read_strings(&reader, &rows); + ASSERT_EQ(rows.size(), 1); + EXPECT_EQ(rows[0][0], "1"); + if (i > 0) { + EXPECT_EQ(rows[0][1], "6"); + } + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST_F(DeltaReaderTest, WidensNestedStructArrayAndMapValues) { + auto int64_type = make_nullable(std::make_shared()); + columns = { + typed_column(0, "struct", + std::make_shared(DataTypes {int64_type}, Strings {"a"})), + typed_column(1, "map", std::make_shared(int64_type, int64_type)), + typed_column(2, "array", std::make_shared(int64_type))}; + const char* files[] = {"part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet", + "part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet"}; + const int64_t max = std::numeric_limits::max(); + const std::vector> expected { + {1, 2, 3, 4, 5}, {max, max, max, max, std::numeric_limits::min()}}; + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + const auto value = [](const IColumn& column, size_t row) { + const auto& nullable_column = assert_cast(column); + EXPECT_FALSE(nullable_column.is_null_at(row)); + return assert_cast(nullable_column.get_nested_column()) + .get_element(row); + }; + for (size_t i = 0; i < 2; ++i) { + SCOPED_TRACE(files[i]); + ASSERT_TRUE(reader.prepare_split(split(table_params(), files[i])).ok()); + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + bool eos = false; + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(block.rows(), 1); + const auto& structure = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + const auto& map = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + const auto& array = assert_cast( + assert_cast(*block.get_by_position(2).column) + .get_nested_column()); + EXPECT_EQ(map.get_offsets()[0], 1); + EXPECT_EQ(array.get_offsets()[0], 2); + EXPECT_EQ(value(structure.get_column(0), 0), expected[i][0]); + EXPECT_EQ(value(map.get_keys(), 0), expected[i][1]); + EXPECT_EQ(value(map.get_values(), 0), expected[i][2]); + EXPECT_EQ(value(array.get_data(), 0), expected[i][3]); + EXPECT_EQ(value(array.get_data(), 1), expected[i][4]); + // FileScannerV2 drains the current split to EOS before preparing the next one. + while (!eos) { + status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(block.rows(), 0); + } + } + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, TimestampUsesSessionZoneForInt96MicrosAndMillis) { + columns = {typed_column(0, "time", std::make_shared(6))}; + const char* files[] = {"part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet", + "part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet", + "part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet"}; + for (const auto* zone : {"UTC", "Asia/Shanghai"}) { + state.set_timezone(zone); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + const std::string expected = std::string(zone) == "UTC" ? "2020-02-01 08:09:10.000000" + : "2020-02-01 16:09:10.000000"; + for (const auto* file : files) { + SCOPED_TRACE(file); + ASSERT_TRUE(reader.prepare_split(split(table_params(), file)).ok()); + std::vector> rows; + read_strings(&reader, &rows); + ASSERT_EQ(rows.size(), 1); + EXPECT_EQ(rows[0][0], expected); + } + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST_F(DeltaReaderTest, TimestampNtzPreservesWallClockAcrossSessionZones) { + columns = {typed_column(0, "tsntz", std::make_shared(6))}; + for (const auto* zone : {"UTC", "Asia/Shanghai", "America/Los_Angeles"}) { + state.set_timezone(zone); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + ASSERT_TRUE(reader.prepare_split(split(table_params(), + "part-00000-6240e68e-2304-449a-a1e6-0e24866d3508." + "c000.snappy.parquet")) + .ok()); + std::vector> rows; + read_strings(&reader, &rows); + ASSERT_EQ(rows.size(), 1); + EXPECT_EQ(rows[0][0], "2021-11-18 02:30:00.123456"); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST_F(DeltaReaderTest, DiskVectorCacheAvoidsSecondRead) { + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + auto options = split(table_params(disk_dv(2))); + ASSERT_TRUE(reader._parse_delete_predicates(options).ok()); + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheMissCount")->value(), 1); + FailDeletionVectorIO fail_io; + ASSERT_TRUE(reader._parse_delete_predicates(options).ok()); + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheHitCount")->value(), 1); + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, DeletionVectorsUseFileGlobalPositionsAcrossByteRanges) { + const auto path = (std::filesystem::path(std::getenv("ROOT")) / + "be/ut_build_ASAN/deltalake_multiple_ranges.parquet") + .string(); + arrow::Int64Builder builder; + for (int64_t id = 0; id < 30; ++id) { + ASSERT_TRUE(builder.Append(id).ok()); + } + std::shared_ptr ids; + ASSERT_TRUE(builder.Finish(&ids).ok()); + auto table = arrow::Table::Make(arrow::schema({arrow::field("id", arrow::int64())}), {ids}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.disable_dictionary(); + ASSERT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, 5, + properties.build()) + .ok()); + ASSERT_TRUE((*output)->Close().ok()); + const auto metadata = ::parquet::ParquetFileReader::OpenFile(path)->metadata(); + ASSERT_EQ(metadata->num_row_groups(), 6); + + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + for (const auto& dv : {disk_dv(2), inline_dv(2)}) { + for (int group = 0; group < metadata->num_row_groups(); ++group) { + SCOPED_TRACE(group); + const auto chunk = metadata->RowGroup(group)->ColumnChunk(0); + auto options = split(table_params(dv), path.c_str()); + options.current_range.__set_start_offset(chunk->data_page_offset()); + options.current_range.__set_size(chunk->total_compressed_size()); + ASSERT_TRUE(reader.prepare_split(options).ok()); + std::vector> actual; + read(&reader, &actual); + std::vector> expected; + for (int64_t id = group * 5; id < (group + 1) * 5; ++id) { + if (id != 0 && id != 7 && id != 14) { + expected.emplace_back(id); + } + } + EXPECT_EQ(actual, expected); + } + } + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheMissCount")->value(), 2); + EXPECT_EQ(profile.get_counter("DeletionVectorDecodedCacheHitCount")->value(), 10); + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, MapsPhysicalNameToRenamedLogicalColumn) { + columns = {table_column(7, "renamed")}; + scan_params.__set_current_schema_id(42); + scan_params.__set_history_schema_info({schema(42, {schema_field("id", 7)})}); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + ASSERT_TRUE(reader.prepare_split(split(table_params(std::nullopt, 42))).ok()); + EXPECT_EQ(reader.mapping_mode(), TableColumnMappingMode::BY_FIELD_ID); + std::vector> ids; + read(&reader, &ids); + ASSERT_EQ(ids.size(), 50); + for (int64_t i = 0; i < 50; ++i) { + EXPECT_EQ(ids[i], i); + } + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, ReadsSparkFieldIdsAfterRootAndNestedRenames) { + auto leaf = typed_column(16, "renamed_leaf", std::make_shared()); + auto inner = typed_column( + 15, "renamed_inner", + std::make_shared(DataTypes {leaf.type}, Strings {leaf.name})); + inner.children = {leaf}; + auto outer = typed_column( + 13, "renamed_outer", + std::make_shared(DataTypes {inner.type}, Strings {inner.name})); + outer.children = {inner}; + columns = {table_column(4, "renamed_long"), outer}; + auto physical_inner = schema_field("col-94e598c8-0480-4710-a288-fc332ed449de", 15); + schema::external::TStructField inner_fields; + inner_fields.__set_fields({schema_field("col-75cd857b-0152-438b-a0b3-43db0d4daa55", 16)}); + physical_inner.field_ptr->nestedField.__set_struct_field(inner_fields); + physical_inner.field_ptr->__isset.nestedField = true; + auto physical_outer = schema_field("col-ee8dc621-b36b-4784-bdb5-26c5c364e254", 13); + schema::external::TStructField outer_fields; + outer_fields.__set_fields({physical_inner}); + physical_outer.field_ptr->nestedField.__set_struct_field(outer_fields); + physical_outer.field_ptr->__isset.nestedField = true; + scan_params.__set_current_schema_id(42); + scan_params.__set_history_schema_info({schema( + 42, {schema_field("col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7", 4), physical_outer})}); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + ASSERT_TRUE( + reader + .prepare_split(split( + table_params(std::nullopt, 42), + "part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet")) + .ok()); + std::vector values; + bool eos = false; + while (!eos) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status; + const auto& outer_column = + assert_cast(*block.get_by_position(1).column); + const auto& inner_column = assert_cast( + assert_cast(outer_column.get_nested_column()).get_column(0)); + const auto& leaf_column = assert_cast( + assert_cast(inner_column.get_nested_column()).get_column(0)); + const auto& long_column = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < block.rows(); ++row) { + EXPECT_FALSE(outer_column.is_null_at(row)); + EXPECT_FALSE(inner_column.is_null_at(row)); + EXPECT_FALSE(leaf_column.is_null_at(row)); + EXPECT_FALSE(long_column.is_null_at(row)); + const auto value = assert_cast(long_column.get_nested_column()) + .get_element(row); + EXPECT_EQ(assert_cast(leaf_column.get_nested_column()) + .get_element(row), + value); + values.push_back(value); + } + } + EXPECT_EQ(values, (std::vector {0, 4})); + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, ReaddedNameDoesNotReadDroppedColumn) { + columns = {table_column(8, "id")}; + scan_params.__set_current_schema_id(42); + scan_params.__set_history_schema_info({schema(42, {schema_field("id", 7)})}); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + ASSERT_TRUE(reader.prepare_split(split(table_params(std::nullopt, 42))).ok()); + std::vector> ids; + read(&reader, &ids); + ASSERT_EQ(ids.size(), 50); + for (const auto& id : ids) { + EXPECT_FALSE(id.has_value()); + } + ASSERT_TRUE(reader.close().ok()); +} + +TEST_F(DeltaReaderTest, AnnotatesNestedPhysicalNames) { + auto parent = schema_field("physical_parent", 10); + schema::external::TStructField children; + children.__set_fields({schema_field("physical_child", 11)}); + parent.field_ptr->nestedField.__set_struct_field(children); + parent.field_ptr->__isset.nestedField = true; + scan_params.__set_current_schema_id(42); + scan_params.__set_history_schema_info({schema(42, {parent})}); + deltalake::DeltaReader reader; + ASSERT_TRUE(init(&reader).ok()); + reader._split_schema_id = 42; + ColumnDefinition child = table_column(0, "physical_child"); + ColumnDefinition file_column; + file_column.name = "physical_parent"; + file_column.type = + std::make_shared(DataTypes {child.type}, Strings {child.name}); + file_column.children = {child}; + std::vector file_schema {file_column}; + ASSERT_TRUE(reader.annotate_file_schema(&file_schema).ok()); + EXPECT_EQ(file_schema[0].identifier.get(), 10); + EXPECT_EQ(file_schema[0].children[0].identifier.get(), 11); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(DeltaDeletionVectorTest, ReadsPortableSparkVectorsAndRejectsCorruption) { + for (size_t i = 0; i < 3; ++i) { + const auto file = read_fixture(DV_FILES[i]); + ASSERT_EQ(file.size(), 43 + i * 2); + const auto framed = file.substr(1); + DeletionVector bitmap; + ASSERT_TRUE( + decode_iceberg_deletion_vector_buffer(framed.data(), framed.size(), &bitmap).ok()); + ASSERT_EQ(bitmap.cardinality(), i + 1); + for (size_t n = 0; n <= i; ++n) { + EXPECT_TRUE(bitmap.contains(static_cast(n * 7))); + } + for (size_t offset : {size_t {0}, size_t {4}, framed.size() - 1}) { + auto corrupt = framed; + corrupt[offset] ^= 1; + DeletionVector rejected; + auto status = decode_iceberg_deletion_vector_buffer(corrupt.data(), corrupt.size(), + &rejected); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(rejected.isEmpty()); + } + DeletionVector inline_bitmap; + const auto payload = framed.substr(4, framed.size() - 8); + ASSERT_TRUE(decode_portable_deletion_vector_payload(payload.data(), payload.size(), + &inline_bitmap) + .ok()); + EXPECT_EQ(inline_bitmap, bitmap); + } +} + +TEST(DeltaDeletionVectorTest, RejectsInvalidInlineAndDiskDescriptors) { + deltalake::DeltaReader reader; + DeleteFileDesc desc; + bool has_delete = false; + auto dv = inline_dv(0); + dv.__set_size_in_bytes(dv.size_in_bytes + 1); + EXPECT_TRUE(reader._parse_deletion_vector_file(table_params(dv), &desc, &has_delete) + .is()); + EXPECT_FALSE(has_delete); + dv = disk_dv(0); + dv.__set_cardinality(-1); + EXPECT_TRUE(reader._parse_deletion_vector_file(table_params(dv), &desc, &has_delete) + .is()); + size_t bytes = 0; + EXPECT_TRUE(validate_delta_deletion_vector_read_range(-1, 34, bytes) + .is()); + EXPECT_TRUE(validate_delta_deletion_vector_read_range(0, -1, bytes) + .is()); + EXPECT_TRUE(validate_delta_deletion_vector_read_range(std::numeric_limits::max(), 34, + bytes) + .is()); + EXPECT_TRUE( + validate_delta_deletion_vector_read_range(0, std::numeric_limits::max(), bytes) + .is()); + ASSERT_TRUE(validate_delta_deletion_vector_read_range(1, 34, bytes).ok()); + EXPECT_EQ(bytes, 42); +} + +TEST(DeltaDeletionVectorTest, CacheKeysDistinguishRangesAndInlinePayloads) { + deltalake::DeltaReader reader; + auto dv = disk_dv(0); + DeleteFileDesc first; + bool has_delete; + ASSERT_TRUE(reader._parse_deletion_vector_file(table_params(dv), &first, &has_delete).ok()); + EXPECT_TRUE(has_delete); + for (int field = 0; field < 2; ++field) { + auto other = dv; + if (field == 0) { + other.__set_offset(2); + } else { + other.__set_size_in_bytes(36); + } + DeleteFileDesc changed; + ASSERT_TRUE(reader._parse_deletion_vector_file(table_params(other), &changed, &has_delete) + .ok()); + EXPECT_NE(first.key, changed.key); + } + DeleteFileDesc inline_first; + DeleteFileDesc inline_second; + ASSERT_TRUE(reader._parse_deletion_vector_file(table_params(inline_dv(0)), &inline_first, + &has_delete) + .ok()); + ASSERT_TRUE(reader._parse_deletion_vector_file(table_params(inline_dv(1)), &inline_second, + &has_delete) + .ok()); + EXPECT_NE(first.key, inline_first.key); + EXPECT_NE(inline_first.key, inline_second.key); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/table/test_data/deltalake/LICENSE.txt b/be/test/format_v2/table/test_data/deltalake/LICENSE.txt new file mode 100644 index 00000000000000..a04059199fb38a --- /dev/null +++ b/be/test/format_v2/table/test_data/deltalake/LICENSE.txt @@ -0,0 +1,388 @@ +Copyright (2021) The Delta Lake Project Authors. All rights reserved. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + + +------------------------------------------------------------------------- +This project includes code derived from the Apache Spark project. +The individual files containing this code carry the original Apache Spark +license, which is reproduced here as well: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/be/test/format_v2/table/test_data/deltalake/README.md b/be/test/format_v2/table/test_data/deltalake/README.md new file mode 100644 index 00000000000000..129f2289b0db88 --- /dev/null +++ b/be/test/format_v2/table/test_data/deltalake/README.md @@ -0,0 +1,47 @@ +# Delta Lake reader fixtures + +Unmodified files from the Delta Lake project's Apache-2.0-licensed golden tables: + +- Release: v4.4.1 +- Commit: `cebdc44dc07d331383e72811177af4ffc8638990` +- Source: `connectors/golden-tables/src/main/resources/golden/log-replay-dv-key-cases` +- Generator: `connectors/golden-tables/src/test/scala/io/delta/golden/GoldenTables.scala` +- Upstream: https://github.com/delta-io/delta/tree/cebdc44dc07d331383e72811177af4ffc8638990/connectors/golden-tables + +The generator writes `spark.range(50).repartition(1)` and then deletes id 0, 7, and +14 in successive commits. The Parquet file contains the original 50 rows. Each +DV starts at offset 1, after the file version byte; its sizeInBytes excludes the +four-byte length prefix and four-byte checksum. Tests also extract the portable +payload to exercise the inline representation without synthesizing a bitmap. + +| File | sizeInBytes | Cardinality | Deleted positions | SHA-256 | +| --- | ---: | ---: | --- | --- | +| `deletion_vector_37d10da3-70a1-4730-bc58-3b44f9617505.bin` | 34 | 1 | 0 | `aa23945a1206ddf0f7551d62468a4d5b34d5d9a55c6a89d1ecfa56337a87cdf2` | +| `deletion_vector_3d8a467a-2fbd-4d35-8e3a-775894a30576.bin` | 36 | 2 | 0, 7 | `6aa4d9879e16533881cb15e8f5ae7b871939dcaf679d4fe14b37112370a3974a` | +| `deletion_vector_d12e7d16-e46d-48c9-8a71-b222c26dfc3b.bin` | 38 | 3 | 0, 7, 14 | `4d91ecfc1e7f6571cd2393ee99f40c49e645db89b81799c0c09a62011492b57e` | +| `part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet` | — | — | — | `23548eea29f5e4c9c57d2aed7a168dcd9732d1617a3fe0c3a524a338b89b2658` | + +Additional unmodified Parquet files from the same commit cover type widening and +timestamp semantics. Expected rows were cross-checked with PyArrow 21.0.0. The +INT96/MICROS/MILLIS files each hold the instant 2020-02-01 08:09:10 UTC; the NTZ +file holds the wall-clock value 2021-11-18 02:30:00.123456. Widening files cover +values before and after schema evolution, including integer bounds and nested +struct/array/map values. + +| Golden-table source path | SHA-256 | +| --- | --- | +| `type-widening/part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet` | `696e42eb3c13413ff054801e661b4325b5661a2fc6289ea01fac5d6baa8cf908` | +| `type-widening/part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet` | `d2b7af9733b4039f159e7a3722aa09ceff2c5199760617f8795f1d8d2b800c1a` | +| `type-widening-nested/part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet` | `109375e0614eb11197629b4f9757d4ef0c17c8bcd94feafe95ec4e7fa58cde8d` | +| `type-widening-nested/part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet` | `f19ff0530f929f936603434c120257593f338b1aede47beb065f517c69562e9e` | +| `kernel-timestamp-INT96/part=2020-01-01 08%3A09%3A10.001/part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet` | `a5a227a34fd9166b33d7bd442237f4b470851452e597df727a77625b787b0211` | +| `kernel-timestamp-TIMESTAMP_MICROS/part=2020-01-01 08%3A09%3A10.001/part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet` | `e347f366786d2736e3d7dcbf5ba89859ade96d333486f417b08d9c64e9205e60` | +| `kernel-timestamp-TIMESTAMP_MILLIS/part=2020-01-01 08%3A09%3A10.001/part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet` | `74b764568c8e19e47d1c9591bdb8e7a4f4f42bbae47b619ebe5aa9b3adb1ddb9` | +| `data-reader-timestamp_ntz/tsNtzPartition=2013-07-05 17%3A01%3A00.123456/part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet` | `50ce09f271845ec07b67abef02a7cdcc5a981025aec93f4ef4da1fadcbba5db9` | +| `table-with-columnmapping-mode-id/part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet` | `7c859554d119d148fa704d7c9a2a31ef6e59c02fe87a78bfd2dc2b977efaf136` | + +The column-mapping fixture carries real Parquet field IDs. Projected field 4 and +nested field 13/15/16 both contain rows 0 and 4; the reader test renames every +projected logical field while retaining those IDs. The byte-range test generates +a separate Parquet file at runtime with six row groups and applies the real DV +above to positions 0, 7, and 14 across row-group boundaries. diff --git a/be/test/format_v2/table/test_data/deltalake/deletion_vector_37d10da3-70a1-4730-bc58-3b44f9617505.bin b/be/test/format_v2/table/test_data/deltalake/deletion_vector_37d10da3-70a1-4730-bc58-3b44f9617505.bin new file mode 100644 index 00000000000000..ff6b720eeb89f4 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/deletion_vector_37d10da3-70a1-4730-bc58-3b44f9617505.bin differ diff --git a/be/test/format_v2/table/test_data/deltalake/deletion_vector_3d8a467a-2fbd-4d35-8e3a-775894a30576.bin b/be/test/format_v2/table/test_data/deltalake/deletion_vector_3d8a467a-2fbd-4d35-8e3a-775894a30576.bin new file mode 100644 index 00000000000000..19e0e053a276e8 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/deletion_vector_3d8a467a-2fbd-4d35-8e3a-775894a30576.bin differ diff --git a/be/test/format_v2/table/test_data/deltalake/deletion_vector_d12e7d16-e46d-48c9-8a71-b222c26dfc3b.bin b/be/test/format_v2/table/test_data/deltalake/deletion_vector_d12e7d16-e46d-48c9-8a71-b222c26dfc3b.bin new file mode 100644 index 00000000000000..06a890d11e0124 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/deletion_vector_d12e7d16-e46d-48c9-8a71-b222c26dfc3b.bin differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet new file mode 100644 index 00000000000000..3ac29e7fd4c66d Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet new file mode 100644 index 00000000000000..db05bc1f76e9bd Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet new file mode 100644 index 00000000000000..df15e0fd540244 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet new file mode 100644 index 00000000000000..6c67447683a9b2 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet new file mode 100644 index 00000000000000..80553911f7c866 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet new file mode 100644 index 00000000000000..9b4f72ae5cbc94 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet new file mode 100644 index 00000000000000..c8ad1d4ac0754e Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet new file mode 100644 index 00000000000000..6f2d7e81a4d0a0 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet new file mode 100644 index 00000000000000..f3ad24fe56f986 Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet differ diff --git a/be/test/format_v2/table/test_data/deltalake/part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet b/be/test/format_v2/table/test_data/deltalake/part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet new file mode 100644 index 00000000000000..2214c0fb45c0da Binary files /dev/null and b/be/test/format_v2/table/test_data/deltalake/part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet differ diff --git a/build.sh b/build.sh index ef53d46c870417..d68356515f1899 100755 --- a/build.sh +++ b/build.sh @@ -832,7 +832,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Keep this list identical to the deploy loop's (search CONN_PLUGIN_DIR). A module missing here # but present there is not a no-op: the deploy step unzips whatever archive is left in the # module's target/ from some earlier build, so the plugin silently ships stale. - for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb; do + for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb deltalake; do if [[ -d "${DORIS_HOME}/fe/fe-connector/fe-connector-${_conn_mod}" ]]; then modules+=("fe-connector/fe-connector-${_conn_mod}") fi @@ -1348,7 +1348,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy connector provider plugins as independent plugin directories. # Each sub-directory is one connector backend loaded at runtime by ConnectorPluginManager. CONN_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/connector" - for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb; do + for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb deltalake; do conn_plugin_target="${CONN_PLUGIN_DIR}/${conn_module}" conn_module_dir="${DORIS_HOME}/fe/fe-connector/fe-connector-${conn_module}" if [ ! -d "${conn_module_dir}" ]; then diff --git a/docker/thirdparties/docker-compose/iceberg/README.md b/docker/thirdparties/docker-compose/iceberg/README.md index ab60fe4a704a29..93660d7b132817 100644 --- a/docker/thirdparties/docker-compose/iceberg/README.md +++ b/docker/thirdparties/docker-compose/iceberg/README.md @@ -209,3 +209,98 @@ bash docker/thirdparties/docker-compose/iceberg/encryption/test_restart.sh \ The test requires the first reset after restart to succeed and then checks the SDK/Spark snapshot results and KMS failure cases again. It never retries reset. + +## Delta Lake regression + +The optional `deltalake` Compose profile reuses this stack's MinIO and the +`apache/spark:4.0.0` image. Delta gets a separate Spark service and Hive 3 HMS: +`DeltaCatalog` must own `spark_catalog`, while the existing Iceberg/Paimon Spark +configuration has its own session catalog and extensions. Delta uses +`delta-spark_4.0_2.13:4.4.1`, the artifact for Spark 4.0. + +Enable it with the standard runner: + +```bash +ENABLE_DELTALAKE_TEST=1 CONTAINER_UID=delta-test- \ + bash docker/thirdparties/run-thirdparties-docker.sh -c iceberg +``` + +Use Bash 4 or newer on macOS. Pass the same `ENABLE_DELTALAKE_TEST=1` when stopping +this stack with `--stop`. With the flag omitted, the existing Iceberg/Paimon +services run without the optional Delta services. + +The runner first prepares the existing Iceberg archive, then calls +`scripts/deltalake/prepare.py`. The Python script downloads and verifies SHA-256 +for all seven Spark jars and the pinned Delta source archive. It caches them in +`cache/deltalake`, extracts 31 selected golden tables, normalizes table directory +names to lower-case underscores, and retains upstream `LICENSE.txt`, `NOTICE.txt` +and a source manifest. A missing table or checksum mismatch fails preparation. +`MAVEN_REPOSITORY_URL` can select a Maven mirror; pinned hashes remain mandatory. + +Golden tables come from Delta Lake v4.4.1, commit +`cebdc44dc07d331383e72811177af4ffc8638990`, under +`connectors/golden-tables/src/main/resources/golden/`, licensed under Apache 2.0. +The archive SHA-256 is +`a9b81e3e0a10c4b32afec823b5a2803eab57c06f542664478ecd438b2b982339`. +The exact selection and artifact hashes live in `scripts/deltalake/prepare.py`. +Golden fixtures are uploaded to the private `delta` bucket under +`warehouse/delta_golden`; the metadata-only `checkpoint` fixture intentionally +has no data objects, and `no_delta_log_folder` is an error-path fixture. + +The Spark service runs the SQL fixtures in +`scripts/create_preinstalled_scripts/deltalake/`, followed by the log-compaction, +golden registration and Parquet pruning helpers. Only after these complete and +Spark Thrift answers a query does the service become healthy. The first successful +setup writes `cache/deltalake/spark/FIXTURES_READY`. Restarts reuse those fixtures +and only restart Spark Thrift. The storage initializer also writes `GOLDEN_READY` +in that state directory after a successful upload, and skips uploading golden +objects on subsequent starts. To deliberately regenerate fixtures after changing +their definitions, explicitly set `DELTALAKE_REGENERATE_FIXTURES=1`; this uploads +the pinned golden objects again and replaces the Spark fixture tables before +populating them. Return the flag to `0` for subsequent +starts. Keep the Spark, HMS and MinIO state directories together when restoring a +retained environment; the marker assumes their corresponding data is still present. +Regression suites retain their catalogs/tables for inspection. Spark uses two local cores +and a 2 GiB driver; allow additional Docker memory when running it alongside the +full Iceberg stack. + +| Compose environment variable | Default | Regression setting | +| --- | --- | --- | +| `MINIO_API_PORT` | 19001 | `deltalakeMinioPort` | +| `DELTALAKE_HMS_PORT` | 19383 | `deltalakeHmsPort` | +| `DELTALAKE_SPARK_PORT` | 11002 | `deltalakeSparkPort` | + +Set these in `regression-test/conf/regression-conf-custom.groovy`, together with +`enableDeltaLakeTest=true`, `externalEnvIp` and +`deltalakeWarehouse="s3a://delta/warehouse"`, then run: + +```bash +bash run-regression-test.sh --run -d external_table_p0/deltalake \ + -s test_deltalake_basic,test_deltalake_skipping,test_deltalake_features,test_deltalake_metadata,test_deltalake_hive,test_deltalake_errors,test_deltalake_parquet,test_deltalake_parquet_deletion_vector,test_deltalake_metrics,test_deltalake_hms_registration +``` + +The tests use both Delta HMS/filesystem catalogs and a Hive catalog's Delta +routing. They cover native reads, mapping/renames, type widening, deletion +vectors, checkpoints/sidecars/compaction, TTL/REFRESH, streaming, COUNT, pruning +and profiles. Spark Thrift is required for the refresh tests' new commits. +The optional million-file fixture is prepared separately; see +`regression-test/suites/external_table_p2/deltalake/README.md`. + +For a Delta-only local run, use the same template and preparation script without +starting the existing Iceberg services or downloading their data archive: + +```bash +python3 docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare.py +cd docker/thirdparties/docker-compose/iceberg +sed 's/doris--/delta-local-/g' iceberg.yaml.tpl > iceberg.yaml +ICEBERG_MINIO_DATA_DIR=./cache/deltalake/minio_data \ + docker compose -p delta-local --env-file iceberg.env -f iceberg.yaml \ + --profile deltalake up -d --wait spark-deltalake +``` + +The MinIO directory override keeps this lightweight run from creating the +`data/` directory that the full runner uses to detect its Iceberg archive. +Use a unique container prefix and unused host ports for each worktree, and keep +the same directory override when restarting this lightweight stack. Its data +and Hive metastore persist in `cache/deltalake`; stop containers before removing +that generated state. diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl index ee12060f2f21a1..292609994d351d 100644 --- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl +++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl @@ -53,6 +53,98 @@ services: timeout: 120s retries: 120 + # Opt in with ENABLE_DELTALAKE_TEST=1 in the runner, or --profile deltalake in Compose. + # DeltaCatalog must own spark_catalog, so Delta has its own Spark session/configuration. + spark-deltalake: + profiles: [deltalake] + image: apache/spark:4.0.0 + container_name: doris--spark-deltalake + user: root + environment: + DELTALAKE_REGENERATE_FIXTURES: ${DELTALAKE_REGENERATE_FIXTURES:-0} + depends_on: + delta-hms: + condition: service_healthy + delta-storage-init: + condition: service_completed_successfully + ports: + - ${DELTALAKE_SPARK_PORT:-11002}:10000 + volumes: + - ./scripts:/mnt/scripts:ro + - ./cache/deltalake/jars:/delta-jars:ro + - ./cache/deltalake/spark:/delta-state + entrypoint: [/bin/bash, /mnt/scripts/deltalake/entrypoint.sh] + networks: + - doris--iceberg + healthcheck: + test: [CMD, test, -f, /delta-state/SUCCESS] + interval: 5s + timeout: 5s + retries: 120 + + delta-postgres: + profiles: [deltalake] + image: postgres:14 + container_name: doris--delta-postgres + environment: + POSTGRES_DB: metastore + POSTGRES_USER: hive + POSTGRES_PASSWORD: hive + # Hive 3's bundled JDBC driver predates PostgreSQL's SCRAM support. Test network only. + POSTGRES_HOST_AUTH_METHOD: trust + volumes: + - ./cache/deltalake/postgres:/var/lib/postgresql/data + networks: + - doris--iceberg + healthcheck: + test: [CMD-SHELL, "pg_isready -U hive -d metastore"] + interval: 5s + timeout: 5s + retries: 60 + + delta-hms: + profiles: [deltalake] + image: doristhirdpartydocker/hive:3.1.2-postgresql-metastore + platform: linux/amd64 + container_name: doris--delta-hms + depends_on: + delta-postgres: + condition: service_healthy + environment: + CORE_CONF_fs_defaultFS: file:/// + HADOOP_CLASSPATH: /opt/hadoop-3.2.1/share/hadoop/tools/lib/* + ports: + - ${DELTALAKE_HMS_PORT:-19383}:9083 + volumes: + - ./scripts/deltalake/hive-site.xml:/opt/hive/conf/hive-site.xml:ro + - ./scripts/deltalake/start-hms.sh:/fixtures/start-hms.sh:ro + - ./cache/deltalake/hms:/state + entrypoint: [/bin/bash, /fixtures/start-hms.sh] + networks: + - doris--iceberg + healthcheck: + test: [CMD, bash, -c, "exec 3<>/dev/tcp/127.0.0.1/9083"] + interval: 5s + timeout: 5s + retries: 120 + + delta-storage-init: + profiles: [deltalake] + image: doristhirdpartydocker/mc:RELEASE.2025-01-17T23-25-50Z + container_name: doris--delta-storage-init + environment: + DELTALAKE_REGENERATE_FIXTURES: ${DELTALAKE_REGENERATE_FIXTURES:-0} + depends_on: + minio: + condition: service_healthy + volumes: + - ./cache/deltalake/golden:/golden:ro + - ./cache/deltalake/spark:/delta-state + - ./scripts/deltalake/init-storage.sh:/fixtures/init-storage.sh:ro + entrypoint: [/bin/sh, /fixtures/init-storage.sh] + networks: + - doris--iceberg + postgres: image: ${ICEBERG_POSTGRES_IMAGE:-postgis/postgis:14-3.3} container_name: doris--iceberg-postgres @@ -172,7 +264,7 @@ services: - MINIO_ROOT_PASSWORD=password - MINIO_DOMAIN=minio volumes: - - ./data/input/minio_data:/data + - ${ICEBERG_MINIO_DATA_DIR:-./data/input/minio_data}:/data - ./scripts/preinstalled_data/:/mnt/preinstalled_data networks: doris--iceberg: diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/00-basic.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/00-basic.sql new file mode 100644 index 00000000000000..99450237b2fa30 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/00-basic.sql @@ -0,0 +1,57 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +CREATE DATABASE IF NOT EXISTS delta_m1 LOCATION 's3a://delta/warehouse/delta_m1'; +DROP TABLE IF EXISTS delta_m1.basic; +CREATE TABLE delta_m1.basic (id BIGINT, value STRING, part STRING) +USING delta PARTITIONED BY (part) +LOCATION 's3a://delta/warehouse/delta_m1/basic' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); +INSERT OVERWRITE delta_m1.basic VALUES (1, 'first', 'a'), (2, 'second', 'b'), (3, 'third', NULL); +SELECT * FROM delta_m1.basic ORDER BY id; +DESCRIBE EXTENDED delta_m1.basic; + +DROP TABLE IF EXISTS delta_m1.types; +CREATE TABLE delta_m1.types ( + id BIGINT, b BOOLEAN, i8 TINYINT, i16 SMALLINT, i32 INT, i64 BIGINT, + f FLOAT, d DOUBLE, dec DECIMAL(20,4), str STRING, bin BINARY, + dt DATE, ts TIMESTAMP, arr ARRAY, m MAP, nested STRUCT +) USING delta LOCATION 's3a://delta/warehouse/delta_m1/types' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); +INSERT OVERWRITE delta_m1.types VALUES + (1, true, -128, 32767, 2147483647, 9223372036854775807, 1.25, -2.5, 123456789012.3400, + '中文 / + %', unhex('00ff1d'), DATE '2024-02-29', TIMESTAMP '2024-03-10 02:30:00.123456', + array(1, NULL, 3), map('a', 9L, 'b', 10L), named_struct('MiXeD', 7, 'Txt', 'nested')), + (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +SELECT * FROM delta_m1.types ORDER BY id; + +DROP TABLE IF EXISTS delta_m1.empty; +CREATE TABLE delta_m1.empty (id BIGINT) USING delta +LOCATION 's3a://delta/warehouse/delta_m1/empty' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); + +DROP TABLE IF EXISTS delta_m1.partition_types; +CREATE TABLE delta_m1.partition_types (id BIGINT, p_bool BOOLEAN, p_int INT, + p_dec DECIMAL(10,2), p_date DATE, p_ts TIMESTAMP, p_str STRING) +USING delta PARTITIONED BY (p_bool,p_int,p_dec,p_date,p_ts,p_str) +LOCATION 's3a://delta/warehouse/delta_m1/partition_types' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); +INSERT OVERWRITE delta_m1.partition_types VALUES + (1, true, 42, 12.50, DATE '2024-02-29', TIMESTAMP '2024-03-10 02:30:00.123456', 'a/b=c%20+d 中文'), + (2, false, -5, -9.00, DATE '1969-12-31', TIMESTAMP '1969-12-31 23:59:59.123456', '__HIVE_DEFAULT_PARTITION__'), + (3, NULL, NULL, NULL, NULL, NULL, NULL); +SELECT * FROM delta_m1.partition_types ORDER BY id; diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/01-skipping.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/01-skipping.sql new file mode 100644 index 00000000000000..744e2f50b1d2d0 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/01-skipping.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP TABLE IF EXISTS delta_m1.skipping; +CREATE TABLE delta_m1.skipping (id BIGINT, s STRING, dec DECIMAL(20,4), f FLOAT, d DOUBLE) +USING delta LOCATION 's3a://delta/warehouse/delta_m1/skipping' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); +INSERT OVERWRITE delta_m1.skipping SELECT /*+ COALESCE(1) */ + id, concat('low-', id), cast(id AS DECIMAL(20,4)), cast(id AS FLOAT), cast(id AS DOUBLE) FROM range(0, 10); +INSERT INTO delta_m1.skipping SELECT /*+ COALESCE(1) */ + id, concat('high-', id), cast(id AS DECIMAL(20,4)), cast(id AS FLOAT), cast(id AS DOUBLE) FROM range(100, 110); +INSERT INTO delta_m1.skipping VALUES (NULL, NULL, NULL, NULL, NULL); +SELECT * FROM delta_m1.skipping ORDER BY id; +SELECT count(*) FROM delta_m1.skipping; +SELECT * FROM delta_m1.skipping WHERE id < 5 ORDER BY id; +SELECT * FROM delta_m1.skipping WHERE NOT (id <=> 1) ORDER BY id; diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/02-features.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/02-features.sql new file mode 100644 index 00000000000000..77be31a2cf98fb --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/02-features.sql @@ -0,0 +1,109 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +CREATE OR REPLACE TABLE delta_m1.mapping_name ( + id BIGINT, old STRING, nested STRUCT, + arr ARRAY>, m MAP>, part STRING) +USING delta PARTITIONED BY (part) +LOCATION 's3a://delta/warehouse/delta_m1/mapping_name' +TBLPROPERTIES ('delta.columnMapping.mode'='name', 'delta.enableDeletionVectors'='false'); +INSERT INTO delta_m1.mapping_name VALUES + (1, 'old-value', named_struct('Before', 10, 'Keep', 'old-nested'), + array(named_struct('Before', 100)), map('k', named_struct('Before', 1000)), 'a/b'); +ALTER TABLE delta_m1.mapping_name RENAME COLUMN id TO renamed; +ALTER TABLE delta_m1.mapping_name RENAME COLUMN nested.Before TO After; +ALTER TABLE delta_m1.mapping_name RENAME COLUMN arr.element.Before TO After; +ALTER TABLE delta_m1.mapping_name RENAME COLUMN m.value.Before TO After; +ALTER TABLE delta_m1.mapping_name RENAME COLUMN part TO partition_renamed; +ALTER TABLE delta_m1.mapping_name DROP COLUMN old; +ALTER TABLE delta_m1.mapping_name ADD COLUMNS (old STRING); +ALTER TABLE delta_m1.mapping_name DROP COLUMN nested.Keep; +ALTER TABLE delta_m1.mapping_name ADD COLUMNS (nested.Keep STRING); +INSERT INTO delta_m1.mapping_name (renamed, nested, arr, m, partition_renamed, old) VALUES + (2, named_struct('After', 20, 'Keep', 'new-nested'), array(named_struct('After', 200)), + map('k', named_struct('After', 2000)), NULL, 'new-value'); +SELECT * FROM delta_m1.mapping_name ORDER BY renamed; + +CREATE OR REPLACE TABLE delta_m1.mapping_id ( + id BIGINT, old STRING, nested STRUCT, + arr ARRAY>, m MAP>, part STRING) +USING delta PARTITIONED BY (part) +LOCATION 's3a://delta/warehouse/delta_m1/mapping_id' +TBLPROPERTIES ('delta.columnMapping.mode'='id', 'delta.enableDeletionVectors'='false'); +INSERT INTO delta_m1.mapping_id VALUES + (1, 'old-value', named_struct('Before', 10, 'Keep', 'old-nested'), + array(named_struct('Before', 100)), map('k', named_struct('Before', 1000)), 'a/b'); +ALTER TABLE delta_m1.mapping_id RENAME COLUMN id TO renamed; +ALTER TABLE delta_m1.mapping_id RENAME COLUMN nested.Before TO After; +ALTER TABLE delta_m1.mapping_id RENAME COLUMN arr.element.Before TO After; +ALTER TABLE delta_m1.mapping_id RENAME COLUMN m.value.Before TO After; +ALTER TABLE delta_m1.mapping_id RENAME COLUMN part TO partition_renamed; +ALTER TABLE delta_m1.mapping_id DROP COLUMN old; +ALTER TABLE delta_m1.mapping_id ADD COLUMNS (old STRING); +ALTER TABLE delta_m1.mapping_id DROP COLUMN nested.Keep; +ALTER TABLE delta_m1.mapping_id ADD COLUMNS (nested.Keep STRING); +INSERT INTO delta_m1.mapping_id (renamed, nested, arr, m, partition_renamed, old) VALUES + (2, named_struct('After', 20, 'Keep', 'new-nested'), array(named_struct('After', 200)), + map('k', named_struct('After', 2000)), NULL, 'new-value'); +SELECT * FROM delta_m1.mapping_id ORDER BY renamed; + +CREATE OR REPLACE TABLE delta_m1.ntz (id BIGINT, ts TIMESTAMP_NTZ, part TIMESTAMP_NTZ) +USING delta PARTITIONED BY (part) LOCATION 's3a://delta/warehouse/delta_m1/ntz' +TBLPROPERTIES ('delta.enableDeletionVectors'='false'); +INSERT INTO delta_m1.ntz VALUES + (1, TIMESTAMP_NTZ '2024-03-10 02:30:00.123456', TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'), + (2, TIMESTAMP_NTZ '1969-12-31 23:59:59.123456', NULL); +SELECT * FROM delta_m1.ntz ORDER BY id; + +CREATE OR REPLACE TABLE delta_m1.widening ( + id BIGINT, tiny TINYINT, small SMALLINT, int_long INT, int_double INT, int_decimal INT, + f FLOAT, dec DECIMAL(10,2), dt DATE, nested STRUCT, arr ARRAY, m MAP) +USING delta LOCATION 's3a://delta/warehouse/delta_m1/widening' +TBLPROPERTIES ('delta.enableTypeWidening'='true', 'delta.enableDeletionVectors'='false', + 'delta.feature.timestampNtz'='supported'); +INSERT INTO delta_m1.widening VALUES + (1, -128, 32767, 2147483647, -2147483648, 123456789, 1.25, 1234.56, DATE '2024-02-29', + named_struct('s', 7S), array(8S), map(9, 10S)); +ALTER TABLE delta_m1.widening ALTER COLUMN tiny TYPE SMALLINT; +ALTER TABLE delta_m1.widening ALTER COLUMN small TYPE BIGINT; +ALTER TABLE delta_m1.widening ALTER COLUMN int_long TYPE BIGINT; +ALTER TABLE delta_m1.widening ALTER COLUMN int_double TYPE DOUBLE; +ALTER TABLE delta_m1.widening ALTER COLUMN int_decimal TYPE DECIMAL(20,4); +ALTER TABLE delta_m1.widening ALTER COLUMN f TYPE DOUBLE; +ALTER TABLE delta_m1.widening ALTER COLUMN dec TYPE DECIMAL(20,4); +ALTER TABLE delta_m1.widening ALTER COLUMN dt TYPE TIMESTAMP_NTZ; +ALTER TABLE delta_m1.widening ALTER COLUMN nested.s TYPE BIGINT; +ALTER TABLE delta_m1.widening ALTER COLUMN arr.element TYPE BIGINT; +ALTER TABLE delta_m1.widening ALTER COLUMN m.key TYPE BIGINT; +ALTER TABLE delta_m1.widening ALTER COLUMN m.value TYPE BIGINT; +INSERT INTO delta_m1.widening VALUES + (2, 128, 2147483648L, 2147483648L, 1.5, 123.4567, 2.5, 1.2345, TIMESTAMP_NTZ '2024-02-29 12:34:56.123456', + named_struct('s', 2147483648L), array(2147483648L), map(2147483648L, 2147483648L)); +SELECT * FROM delta_m1.widening ORDER BY id; + +CREATE OR REPLACE TABLE delta_m1.dv (id BIGINT, value STRING, part INT) +USING delta PARTITIONED BY (part) LOCATION 's3a://delta/warehouse/delta_m1/dv' +TBLPROPERTIES ('delta.enableDeletionVectors'='true', 'delta.checkpointInterval'='2'); +INSERT INTO delta_m1.dv SELECT id, concat('value-', id), cast(id % 2 AS INT) FROM range(0, 100); +DELETE FROM delta_m1.dv WHERE id % 7 = 0; +UPDATE delta_m1.dv SET value=concat('updated-', id) WHERE id % 11 = 0; +MERGE INTO delta_m1.dv t USING (SELECT 1L id, 'merged-1' value, 1 part) s ON t.id=s.id +WHEN MATCHED THEN UPDATE SET t.value=s.value +WHEN NOT MATCHED THEN INSERT *; +SELECT * FROM delta_m1.dv ORDER BY id; +SELECT count(*) FROM delta_m1.dv; diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/03-metadata.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/03-metadata.sql new file mode 100644 index 00000000000000..eb7f0d4af30454 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/03-metadata.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +CREATE OR REPLACE TABLE delta_m1.log_compaction (id BIGINT, value BIGINT) +USING delta LOCATION 's3a://delta/warehouse/delta_m1/log_compaction' +TBLPROPERTIES ('delta.enableDeletionVectors'='false', 'delta.checkpointInterval'='10000'); +INSERT INTO delta_m1.log_compaction SELECT id, id FROM range(0, 10); +UPDATE delta_m1.log_compaction SET value=value+100 WHERE id<3; +DELETE FROM delta_m1.log_compaction WHERE id%2=0; +INSERT INTO delta_m1.log_compaction VALUES (20, 200), (21, 210), (22, 220); +SELECT * FROM delta_m1.log_compaction ORDER BY id; diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/04-prepare-metadata.py b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/04-prepare-metadata.py new file mode 100644 index 00000000000000..749d5c21619ac1 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/04-prepare-metadata.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run after the SQL fixtures and the pinned Delta golden tables have been uploaded.""" + +from pyspark.sql import SparkSession + + +spark = SparkSession.builder.getOrCreate() +try: + jvm = spark.sparkContext._jvm + conf = spark.sparkContext._jsc.hadoopConfiguration() + warehouse = "s3a://delta/warehouse" + path = f"{warehouse}/delta_m1/log_compaction" + engine = jvm.io.delta.kernel.defaults.engine.DefaultEngine.create(conf) + snapshot = jvm.io.delta.kernel.TableManager.loadSnapshot(path).build(engine) + version = snapshot.getVersion() + compacted = jvm.org.apache.hadoop.fs.Path( + f"{path}/_delta_log/{0:020d}.{version:020d}.compacted.json" + ) + # Kernel 4.4.1 has no public log-compaction entry point. This fixture uses its pinned hook; + # production internal API dependencies remain isolated in DeltaKernelCompat. + if not compacted.getFileSystem(conf).exists(compacted): + jvm.io.delta.kernel.internal.hook.LogCompactionHook( + jvm.io.delta.kernel.internal.fs.Path(path), + jvm.io.delta.kernel.internal.fs.Path(f"{path}/_delta_log"), + 0, version, 0, + ).threadSafeInvoke(engine) + print(f"Prepared log compaction through version {version}", flush=True) + + spark.sql(f"CREATE DATABASE IF NOT EXISTS delta_golden LOCATION '{warehouse}/delta_golden'") + tables = [ + "table_with_columnmapping_mode_name", "table_with_columnmapping_mode_id", + "data_reader_timestamp_ntz", "data_reader_timestamp_ntz_name_mode", "data_reader_timestamp_ntz_id_mode", + "type_widening", "type_widening_nested", + "dv_partitioned_with_checkpoint", "dv_with_columnmapping", "log_replay_dv_key_cases", + "multi_part_checkpoint", "only_checkpoint_files", "v2_checkpoint_json", "v2_checkpoint_parquet", + "basic_with_vacuum_protocol_check_feature", + ] + for table in tables: + spark.sql(f"CREATE TABLE IF NOT EXISTS delta_golden.{table} USING delta " + f"LOCATION '{warehouse}/delta_golden/{table}'") +finally: + spark.stop() diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/05-parquet-pruning.py b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/05-parquet-pruning.py new file mode 100644 index 00000000000000..38b86946c8344e --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/05-parquet-pruning.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One file with multiple row groups and pages, including a mapped/renamed column.""" + +from pyspark.sql import SparkSession + + +spark = SparkSession.builder.getOrCreate() +try: + # Delta does not forward arbitrary DataFrameWriter options to ParquetOutputFormat. + conf = spark.sparkContext._jsc.hadoopConfiguration() + for key, value in { + "parquet.block.size": "65536", + "parquet.page.size": "1024", + "parquet.page.row.count.limit": "100", + "parquet.enable.dictionary": "false", + }.items(): + conf.set(key, value) + for mode in ("none", "name", "id"): + table = f"delta_m1.parquet_pruning_{mode}" + path = f"s3a://delta/warehouse/delta_m1/parquet_pruning_{mode}" + spark.sql(f"""CREATE OR REPLACE TABLE {table} (id BIGINT, payload STRING) + USING delta LOCATION '{path}' + TBLPROPERTIES ('delta.enableDeletionVectors'='false', + 'delta.columnMapping.mode'='{mode}')""") + (spark.range(0, 10000, 1, 1).selectExpr("id", "repeat('x', 64) AS payload") + .write.format("delta").mode("overwrite") + .save(path)) + if mode != "none": + spark.sql(f"ALTER TABLE {table} RENAME COLUMN id TO renamed") + column = "id" if mode == "none" else "renamed" + spark.sql(f"SELECT {column}, length(payload) FROM {table} " + f"WHERE {column} >= 100 AND {column} < 120 ORDER BY {column}").show() + # Deleted rows sit inside the ranges the regression predicates keep, in early and late row groups + # and at both ends of the table, so skipped pages or row groups cannot hide a wrong row position. + table = "delta_m1.parquet_pruning_dv" + path = "s3a://delta/warehouse/delta_m1/parquet_pruning_dv" + spark.sql(f"""CREATE OR REPLACE TABLE {table} (id BIGINT, payload STRING) + USING delta LOCATION '{path}' + TBLPROPERTIES ('delta.enableDeletionVectors'='true')""") + # A row-specific payload makes a lazy column read at the wrong position visible after pruning. + (spark.range(0, 10000, 1, 1).selectExpr("id", "rpad(concat('row-', cast(id as string)), 64, 'x') AS payload") + .write.format("delta").mode("overwrite") + .save(path)) + spark.sql(f"DELETE FROM {table} WHERE id IN (0, 5, 101, 105, 119, 199, 200, 9005, 9019, 9999)") + spark.sql(f"SELECT id FROM {table} WHERE id >= 100 AND id < 120 ORDER BY id").show() +finally: + spark.stop() diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/generate_planning_fixture.py b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/generate_planning_fixture.py new file mode 100644 index 00000000000000..d2f0cfa00ba5c4 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/generate_planning_fixture.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate one million synthetic AddFile actions for planning acceptance, without data objects. + +Requires PyArrow and the pinned only-checkpoint-files golden table. Use only metadata +COUNT or the opt-in FE planning test: querying data from this fixture must fail. +""" + +import argparse +import copy +import json +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + + +def generate(golden_table, output): + template = pq.read_table(golden_table / "_delta_log/00000000000000000002.checkpoint.parquet") + rows = template.to_pylist() + metadata = next(row for row in rows if row["metaData"]) + protocol = next(row for row in rows if row["protocol"]) + metadata["metaData"]["id"] = "doris-delta-million-addfiles-planning-only" + metadata["metaData"]["configuration"] = [] + add = next(row for row in rows if row["add"])["add"] + add.update(size=527, modificationTime=0, + stats='{"numRecords":1,"minValues":{"id":1},"maxValues":{"id":1},"nullCount":{"id":0}}') + log = output / "_delta_log" + log.mkdir(parents=True, exist_ok=True) + checkpoint = log / "00000000000000000010.checkpoint.parquet" + with pq.ParquetWriter(checkpoint, template.schema, compression="snappy") as writer: + writer.write_table(pa.Table.from_pylist([metadata, protocol], schema=template.schema)) + for start in range(0, 1_000_000, 10000): + batch = [{"add": dict(add, path=f"planning-only-{i:08}.parquet")} + for i in range(start, start + 10000)] + writer.write_table(pa.Table.from_pylist(batch, schema=template.schema)) + (log / "_last_checkpoint").write_text(json.dumps({ + "version": 10, "size": 1_000_002, "sizeInBytes": checkpoint.stat().st_size, + "numOfAddFiles": 1_000_000, + })) + # Keep the checkpoint's own commit for timestamp resolution; older JSON commits are retired. + meta = copy.deepcopy(metadata["metaData"]) + meta["configuration"] = {} + meta["format"]["options"] = {} + actions = [ + {"commitInfo": {"timestamp": 1700000000000}}, + {"protocol": {"minReaderVersion": 1, "minWriterVersion": 2}}, + {"metaData": meta}, + ] + (log / "00000000000000000010.json").write_text( + "".join(json.dumps(action) + "\n" for action in actions)) + print(f"Created planning-only fixture at {output}: 1,000,000 AddFiles, " + f"{checkpoint.stat().st_size} checkpoint bytes; data objects deliberately absent") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--golden-table", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + generate(args.golden_table, args.output) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/entrypoint.sh b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/entrypoint.sh new file mode 100644 index 00000000000000..82bbde18892d62 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/entrypoint.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +rm -f /delta-state/SUCCESS +cp /delta-jars/*.jar /opt/spark/jars/ +config=/mnt/scripts/deltalake/spark-defaults.conf +fixtures=/mnt/scripts/create_preinstalled_scripts/deltalake +case "${DELTALAKE_REGENERATE_FIXTURES:-0}" in + 0|1) ;; + *) echo "DELTALAKE_REGENERATE_FIXTURES must be 0 or 1" >&2; exit 1 ;; +esac +if [[ ! -f /delta-state/FIXTURES_READY || "${DELTALAKE_REGENERATE_FIXTURES:-0}" == 1 ]]; then + rm -f /delta-state/FIXTURES_READY + # Use one session for SQL fixtures; each Python helper owns and closes its Spark session. + for fixture in "${fixtures}"/*.sql; do + printf 'source %s;\n' "${fixture}" + done > /tmp/deltalake-fixtures.sql + /opt/spark/bin/spark-sql --properties-file "${config}" -f /tmp/deltalake-fixtures.sql + /opt/spark/bin/spark-submit --properties-file "${config}" "${fixtures}/04-prepare-metadata.py" + /opt/spark/bin/spark-submit --properties-file "${config}" "${fixtures}/05-parquet-pruning.py" + touch /delta-state/FIXTURES_READY +else + echo "Reusing retained Delta fixtures; skipping data generation" +fi +/opt/spark/sbin/start-thriftserver.sh --properties-file "${config}" \ + --hiveconf hive.server2.thrift.port=10000 +for attempt in $(seq 1 120); do + if /opt/spark/bin/beeline -u jdbc:hive2://localhost:10000/default -n hadoop \ + -e 'SELECT 1' >/tmp/delta-thrift-ready.log 2>&1; then + touch /delta-state/SUCCESS + exec tail -f /dev/null + fi + sleep 1 +done +cat /tmp/delta-thrift-ready.log >&2 +exit 1 diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/generate_benchmark_parquet.py b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/generate_benchmark_parquet.py new file mode 100644 index 00000000000000..de505f2b94011c --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/generate_benchmark_parquet.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert SF1 TPC tool output to shared Hive/Delta Parquet files (DuckDB 1.4.4).""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import shutil +import tempfile + +import duckdb + + +def convert(kind, generator, data_dir, output): + ddl = generator / ("dss.ddl" if kind == "tpch" else "tpcds.sql") + extension = "tbl" if kind == "tpch" else "dat" + manifest = {} + with tempfile.TemporaryDirectory(dir=output) as temporary: + connection = duckdb.connect(config={"threads": "4", "memory_limit": "2GB", + "temp_directory": temporary}) + try: + tables = re.findall(r"create\s+table\s+(\w+)\s*\((.*?)\);", ddl.read_text(), re.I | re.S) + for table, body in tables: + table = table.lower() + if table == "dbgen_version": + continue + columns = re.findall( + r"(?:^|,\s*)\s*(\w+)\s+(integer|date|decimal\s*\(\s*\d+\s*,\s*\d+\s*\)" + r"|(?:var)?char\(\d+\))", body, re.I) + types = {name.lower(): re.sub(r"(?:var)?char\(\d+\)", "VARCHAR", dtype, flags=re.I) + for name, dtype in columns} + # TPC tool rows have a trailing separator; discard that empty column explicitly. + types["_trailing"] = "VARCHAR" + source = data_dir / f"{table}.{extension}" + if kind == "tpcds": + # The 3.2.0 country distributions include Latin-1 names such as RÉUNION. + utf8 = Path(temporary) / source.name + with source.open(encoding="latin-1") as reader, utf8.open("w", encoding="utf-8") as writer: + shutil.copyfileobj(reader, writer) + source = utf8 + relation = connection.read_csv(str(source), columns=types, header=False, + delimiter="|", quotechar="", escapechar="") + relation = relation.project(",".join(name for name in types if name != "_trailing")) + folder = output / f"delta_{kind}" / table / "data" + folder.mkdir(parents=True, exist_ok=True) + parquet = folder / "part-00000.parquet" + relation.write_parquet(str(parquet), compression="snappy", row_group_size=122880) + count = connection.execute("SELECT count(*) FROM read_parquet(?)", [str(parquet)]).fetchone()[0] + digest = hashlib.sha256() + with parquet.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + manifest[table] = {"rows": count, "bytes": parquet.stat().st_size, "sha256": digest.hexdigest()} + print(kind, table, count, flush=True) + finally: + connection.close() + assert len(manifest) == (8 if kind == "tpch" else 24), (kind, len(manifest)) + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for workload in ["tpch", "tpcds"]: + parser.add_argument(f"--{workload}-generator-dir", type=Path, required=True) + parser.add_argument(f"--{workload}-data-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists() and any(args.output.iterdir()): + parser.error("Output already contains files; reuse its fixtures for tests or choose a new output directory") + args.output.mkdir(parents=True, exist_ok=True) + result = {kind: convert(kind, getattr(args, f"{kind}_generator_dir"), + getattr(args, f"{kind}_data_dir"), args.output) for kind in ["tpch", "tpcds"]} + (args.output / "manifest.json").write_text(json.dumps(result, indent=2) + "\n") diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/hive-site.xml b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/hive-site.xml new file mode 100644 index 00000000000000..0b24031251b9eb --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/hive-site.xml @@ -0,0 +1,33 @@ + + + + javax.jdo.option.ConnectionURLjdbc:postgresql://delta-postgres:5432/metastore + javax.jdo.option.ConnectionDriverNameorg.postgresql.Driver + javax.jdo.option.ConnectionUserNamehive + javax.jdo.option.ConnectionPasswordhive + hive.metastore.schema.verificationtrue + hive.metastore.warehouse.dirfile:///tmp/delta-warehouse + hive.metastore.uristhrift://delta-hms:9083 + fs.s3a.endpointhttp://minio:9000 + fs.s3a.access.keyadmin + fs.s3a.secret.keypassword + fs.s3a.path.style.accesstrue + fs.s3a.connection.ssl.enabledfalse + diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/init-storage.sh b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/init-storage.sh new file mode 100644 index 00000000000000..d4aaff53cd26f7 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/init-storage.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu +case "${DELTALAKE_REGENERATE_FIXTURES:-0}" in + 0|1) ;; + *) echo "DELTALAKE_REGENERATE_FIXTURES must be 0 or 1" >&2; exit 1 ;; +esac +mc alias set delta http://minio:9000 admin password +mc mb --ignore-existing delta/delta +if [ -f /delta-state/GOLDEN_READY ] && [ "${DELTALAKE_REGENERATE_FIXTURES:-0}" = 0 ]; then + echo "Reusing retained Delta golden fixtures; skipping upload" + exit 0 +fi +rm -f /delta-state/GOLDEN_READY +mc mirror --overwrite /golden/ delta/delta/warehouse/delta_golden/ +touch /delta-state/GOLDEN_READY diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare.py b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare.py new file mode 100644 index 00000000000000..8eb06a6fcc4ff3 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Prepare pinned Spark dependencies and Delta golden tables for the optional Compose profile.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import tarfile +import tempfile +import urllib.request + + +DELTA_COMMIT = "cebdc44dc07d331383e72811177af4ffc8638990" +ARCHIVE_SHA256 = "a9b81e3e0a10c4b32afec823b5a2803eab57c06f542664478ecd438b2b982339" +ARTIFACTS = [ + ('io/delta/delta-spark_4.0_2.13/4.4.1/delta-spark_4.0_2.13-4.4.1.jar', + 'd0766b4b0ace348dca7a5851418767e1eea170c7303c4771cda2934f49627ce8'), + ('io/delta/delta-storage/4.4.1/delta-storage-4.4.1.jar', + 'f585818d646cb3625b305270ec24f3e6f0b6ea63ed9bac1ae8f340f8faaeae8f'), + ('io/delta/delta-kernel-api/4.4.1/delta-kernel-api-4.4.1.jar', + '19c93b5e6740bb3001d3deca0b8c3fa5470197c520ffa36ffa58df98e91ae459'), + ('io/delta/delta-kernel-defaults/4.4.1/delta-kernel-defaults-4.4.1.jar', + '329995bbc9b2e5d74a9a7c9e968574c1e438bbb183aaf1b2c629a87ce8558fe2'), + ('io/delta/delta-kernel-unitycatalog/4.4.1/delta-kernel-unitycatalog-4.4.1.jar', + '4d92b9248f8a5f9b3043fea2bb7666f0db26f5491e279c30868280be35450368'), + ('org/apache/hadoop/hadoop-aws/3.4.1/hadoop-aws-3.4.1.jar', + '01334a07e179cae2b7e251f24857f11e3759108c9d6a79dcbe362a4295227da2'), + ('software/amazon/awssdk/bundle/2.24.6/bundle-2.24.6.jar', + '890bf9f2a80488f7a8aa8277d64d13a9ea3a3cdff8607ef3b63a878d8b467b41'), +] + +TABLES = [ + 'table-with-columnmapping-mode-name', + 'table-with-columnmapping-mode-id', + 'data-reader-timestamp_ntz', + 'data-reader-timestamp_ntz-name-mode', + 'data-reader-timestamp_ntz-id-mode', + 'type-widening', + 'type-widening-nested', + 'dv-partitioned-with-checkpoint', + 'dv-with-columnmapping', + 'log-replay-dv-key-cases', + 'checkpoint', + 'multi-part-checkpoint', + 'only-checkpoint-files', + 'v2-checkpoint-json', + 'v2-checkpoint-parquet', + 'basic-with-vacuum-protocol-check-feature', + 'data-reader-primitives', + 'data-reader-nested-struct', + 'data-reader-array-primitives', + 'data-reader-array-complex-objects', + 'data-reader-map', + 'kernel-timestamp-INT96', + 'kernel-timestamp-TIMESTAMP_MICROS', + 'kernel-timestamp-TIMESTAMP_MILLIS', + 'decimal-various-scale-precision', + 'parquet-decimal-dictionaries', + 'parquet-decimal-dictionaries-v1', + 'parquet-decimal-dictionaries-v2', + 'collations-table', + 'spark-variant-checkpoint', + 'no-delta-log-folder', +] + + +def checksum(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download(url, destination, expected): + if destination.is_file() and checksum(destination) == expected: + return + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as temporary: + partial = Path(temporary.name) + try: + print(f"Downloading {destination.name}", flush=True) + with urllib.request.urlopen(url, timeout=120) as response, partial.open("wb") as output: + shutil.copyfileobj(response, output, 1024 * 1024) + actual = checksum(partial) + if actual != expected: + raise ValueError(f"SHA-256 mismatch for {destination.name}: {actual}, expected {expected}") + partial.replace(destination) + finally: + partial.unlink(missing_ok=True) + + +def prepare(output): + output.mkdir(parents=True, exist_ok=True) + maven = os.environ.get("MAVEN_REPOSITORY_URL", "https://repo.maven.apache.org/maven2").rstrip("/") + for artifact, sha256 in ARTIFACTS: + download(f"{maven}/{artifact}", output / "jars" / Path(artifact).name, sha256) + archive = output / "delta-v4.4.1.tar.gz" + download(f"https://codeload.github.com/delta-io/delta/tar.gz/{DELTA_COMMIT}", archive, ARCHIVE_SHA256) + expected_manifest = {"commit": DELTA_COMMIT, "archive_sha256": ARCHIVE_SHA256, "tables": TABLES} + marker = output / "golden-manifest.json" + golden = output / "golden" + if (marker.is_file() and json.loads(marker.read_text()) == expected_manifest + and all((golden / table.lower().replace("-", "_")).is_dir() for table in TABLES)): + print("Delta v4.4.1 dependencies and golden tables are ready", flush=True) + return + prefix = f"delta-{DELTA_COMMIT}/connectors/golden-tables/src/main/resources/golden/" + copied = set() + with tempfile.TemporaryDirectory(dir=output, prefix="golden-") as staging: + staged = Path(staging) / "golden" + staged.mkdir() + with tarfile.open(archive) as bundle: + for member in bundle: + if member.name in (f"delta-{DELTA_COMMIT}/LICENSE.txt", f"delta-{DELTA_COMMIT}/NOTICE.txt"): + destination = Path(staging) / Path(member.name).name + elif member.name.startswith(prefix): + relative = PurePosixPath(member.name[len(prefix):]) + if member.isdir() or relative.parts[0] not in TABLES: + continue + assert ".." not in relative.parts and member.isfile(), member.name + table = relative.parts[0] + copied.add(table) + destination = staged / table.lower().replace("-", "_") / Path(*relative.parts[1:]) + else: + continue + destination.parent.mkdir(parents=True, exist_ok=True) + with bundle.extractfile(member) as source, destination.open("wb") as target: + shutil.copyfileobj(source, target) + missing = set(TABLES) - copied + if missing: + raise ValueError(f"Missing golden tables in pinned archive: {sorted(missing)}") + # This directory contains only generated fixtures, never container state or authored inputs. + if golden.exists(): + shutil.rmtree(golden) + staged.replace(golden) + for name in ("LICENSE.txt", "NOTICE.txt"): + (Path(staging) / name).replace(output / name) + pending_marker = marker.with_suffix(".json.tmp") + pending_marker.write_text(json.dumps(expected_manifest, indent=2) + "\n") + pending_marker.replace(marker) + print(f"Prepared {len(copied)} Delta v4.4.1 golden tables", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, + default=Path(__file__).resolve().parents[2] / "cache/deltalake") + prepare(parser.parse_args().output.resolve()) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare_benchmark.py b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare_benchmark.py new file mode 100644 index 00000000000000..5bcf616ec1908d --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare_benchmark.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Register immutable SF1 Parquet fixtures in HMS and add Delta logs without rewriting data.""" + +import argparse +import json + +from pyspark.sql import SparkSession + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--manifest", required=True) +parser.add_argument("--warehouse", default="s3a://delta/warehouse") +args = parser.parse_args() +with open(args.manifest) as source: + manifest = json.load(source) +spark = SparkSession.builder.getOrCreate() +try: + # This benchmark intentionally reads the same immutable files as plain Parquet and as Delta. + # Disable Spark's format-mismatch guard for the plain Parquet reference on repeat preparation. + spark.conf.set("spark.databricks.delta.formatCheck.enabled", "false") + for workload, tables in manifest.items(): + database = "delta_" + workload + spark.sql(f"CREATE DATABASE IF NOT EXISTS {database}") + for table, info in tables.items(): + path = f"{args.warehouse.rstrip('/')}/{database}/{table}" + spark.sql(f"DROP TABLE IF EXISTS {database}.{table}") + spark.sql(f"CREATE TABLE {database}.{table} USING PARQUET LOCATION '{path}/data'") + parquet_rows = spark.table(f"{database}.{table}").count() + spark.sql(f"CONVERT TO DELTA parquet.`{path}`") + delta_rows = spark.read.format("delta").load(path).count() + assert parquet_rows == delta_rows == info["rows"], (table, parquet_rows, delta_rows, info) + provider = spark._jsparkSession.sessionState().catalog().externalCatalog().getTable( + database, table).provider().get() + assert provider.lower() == "parquet", (table, provider) + print("DELTA_BENCHMARK_FIXTURE", database, table, parquet_rows, provider, flush=True) +finally: + spark.stop() diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/spark-defaults.conf b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/spark-defaults.conf new file mode 100644 index 00000000000000..0390f1ddc567c7 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/spark-defaults.conf @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +spark.master local[2] +spark.driver.memory 2g +spark.sql.shuffle.partitions 2 +spark.sql.session.timeZone UTC +spark.sql.extensions io.delta.sql.DeltaSparkSessionExtension +spark.sql.catalog.spark_catalog org.apache.spark.sql.delta.catalog.DeltaCatalog +spark.sql.catalogImplementation hive +spark.hadoop.hive.metastore.uris thrift://delta-hms:9083 +spark.sql.warehouse.dir s3a://delta/warehouse +spark.hadoop.fs.s3a.endpoint http://minio:9000 +spark.hadoop.fs.s3a.endpoint.region us-east-1 +spark.hadoop.fs.s3a.access.key admin +spark.hadoop.fs.s3a.secret.key password +spark.hadoop.fs.s3a.path.style.access true +spark.hadoop.fs.s3a.connection.ssl.enabled false +spark.hadoop.fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider +spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/start-hms.sh b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/start-hms.sh new file mode 100644 index 00000000000000..e130b1feac8eac --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/deltalake/start-hms.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +export HIVE_AUX_JARS_PATH=/opt/hadoop-3.2.1/share/hadoop/tools/lib +if [[ ! -e /state/schema-initialized ]]; then + /opt/hive/bin/schematool -dbType postgres -initSchema + touch /state/schema-initialized +fi +exec /opt/hive/bin/hive --service metastore -p 9083 diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index af59bbc54e8bf9..af75c8c50ce45f 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -1440,6 +1440,12 @@ start_hive_stack() { start_iceberg() { # iceberg + local -a delta_profile=() + case "${ENABLE_DELTALAKE_TEST:-0}" in + 0) ;; + 1) delta_profile=(--profile deltalake) ;; + *) echo "ENABLE_DELTALAKE_TEST must be 0 or 1" >&2; return 1 ;; + esac ICEBERG_DIR=${ROOT}/docker-compose/iceberg if [[ "${HOST_OS}" == "Darwin" ]]; then export ICEBERG_POSTGRES_IMAGE="${ICEBERG_POSTGRES_IMAGE:-postgres:14}" @@ -1448,7 +1454,8 @@ start_iceberg() { render_uid_template "${ROOT}/docker-compose/iceberg/entrypoint.sh.tpl" "${ROOT}/docker-compose/iceberg/entrypoint.sh" cp "${ROOT}/docker-compose/iceberg/entrypoint.sh" "${ROOT}/docker-compose/iceberg/scripts/entrypoint.sh" register_stack_metadata "iceberg" "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" - compose_down_stack "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" --remove-orphans + compose_cmd "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" \ + "${delta_profile[@]}" down --remove-orphans if [[ "${STOP}" -ne 1 ]]; then # data/ is extracted from the fixture archive once and then reused. A data/ from an older archive lacks # jars that iceberg.yaml now mounts, and Compose would create empty directories in their place, so @@ -1484,7 +1491,11 @@ start_iceberg() { fi bash "${ICEBERG_DIR}/encryption/prepare-jars.sh" - compose_up_stack "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" -d --wait + if [[ "${ENABLE_DELTALAKE_TEST:-0}" -eq 1 ]]; then + python3 "${ICEBERG_DIR}/scripts/deltalake/prepare.py" || return + fi + compose_cmd "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" \ + "${delta_profile[@]}" up -d --wait fi } diff --git a/fe/fe-connector/fe-connector-deltalake/README.md b/fe/fe-connector/fe-connector-deltalake/README.md new file mode 100644 index 00000000000000..32e3832b826edd --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/README.md @@ -0,0 +1,127 @@ +# Delta Lake connector + +The native read-only connector uses Delta Kernel 4.4.1 to load HMS/filesystem tables +and plan native BE Parquet scans. It fixes one snapshot for a statement's schema, +partitions, files and metadata COUNT. Kernel internal API use stays in +`DeltaKernelCompat`. + +## Basic table statistics + +`getTableStatistics` reports logical rows and the sum of active data-file sizes. +The row count is `sum(numRecords - deletionVector.cardinality)` over **all** active +files at the requested version. A missing file record count makes the total row +count unknown (`-1`); an empty active-file set has zero rows. File sizes remain +available when some record counts are missing. Collection reads log/checkpoint +metadata, not Parquet data or deletion-vector bitmap objects. + +On a cold lookup, the connector schedules collection and immediately returns the +current version checksum's `tableSizeBytes`, if available. A checksum has no total +row count; only `numFiles == 0` proves zero rows. Older checksums are not used for +this fallback. The FE can estimate rows from size and Doris row width. Compression, +variable-width values and deletion vectors can make that estimate inaccurate. +Ordinary queries use the FE's standard external-table row-count cache, including +Delta reads through a Hive catalog. The FE caches estimates by table and may +reuse one after a new Delta commit or after background collection finishes. +After `external_cache_refresh_time_minutes` (10 minutes by default), an access +triggers an asynchronous refresh that can pick up the completed count. + +The connector caches statistics by user, database/table/path, table UUID and +snapshot version. Cached tasks use the shared `CatalogMetaCache` ownership and invalidation +rules. At most one load starts per retained entry. The plugin shares two background +workers and a queue of 64 tasks; saturation returns the fallback and is retryable, +never running collection on the query thread. Queued tasks retain coordinates, not +snapshots or AddFile lists. Each worker restores the plugin classloader and storage +authentication through `DeltaLakeRuntime.execute`, reloads the pinned version and +checks its table UUID. Failures are logged and retried on a subsequent lookup. +REFRESH, eviction and catalog close cancel the removed task; a late completion +cannot replace a newly cached task. Cancellation of remote IO still depends on the +filesystem's interrupt/timeout behavior. + +| Catalog property | Default | Meaning | +| --- | --- | --- | +| `meta.cache.deltalake.statistics.enable` | `true` | Enable versioned statistics collection/cache. | +| `meta.cache.deltalake.statistics.ttl-second` | `1800` | Expire after access; `-1` disables time expiry. | +| `meta.cache.deltalake.statistics.capacity` | `1000` | Maximum entries. | +| `meta.cache.deltalake.statistics.max-weight` | unset | Optional estimated-byte limit using the shared cache framework. | + +`enable=false`, `ttl-second=0` or `capacity=0` disables the statistics cache and +background collection, leaving the checksum fallback. ALTER CATALOG rebuilds the +catalog with the new settings. `REFRESH TABLE`, DATABASE and CATALOG invalidate +statistics in the same scope as snapshots. Existing latest-version TTL rules still +control when a new query discovers an external commit. + +Basic row/size statistics are part of M1. Column NDV and histograms are not provided. +Optimizer estimates never feed exact COUNT pushdown: that path independently +requires complete unfiltered metadata and shares only the row-count aggregation +logic. Statistics improve planning inputs, but do not by themselves establish +performance parity with Hive. + +## Small-table scan scheduling + +With external-table batch mode enabled, Delta checks the current checksum's file +count, then a complete count already observed in the statement, then a cached +count for the same user/table identity/version. A known count below +`num_files_in_batch_mode` uses synchronous split planning; large or unknown counts +keep streaming. The scheduling lookup never starts or waits for statistics and +never enumerates files. Background statistics retain one additional file counter, +including when row statistics are missing. Normal exhaustive scans also publish +their complete active-file count into the same cache entry without starting a +statistics task. Partial scans and partition-pruned replay counts cannot establish +the table total. Reports update only the entry captured before scanning, so late +completion cannot resurrect an invalidated entry. A cold snapshot without a checksum +can still stream until a complete count is available. This decision is local +to the Delta plugin, including Delta tables read through an HMS gateway; it does +not change the shared FE row-count cache or BE scanner scheduling. + +## Repeated scans in one statement + +With the existing `enable_external_scan_task_reuse` session variable enabled, +identical synchronous scans share their completed file ranges during planning. +The key includes catalog/query/user, table coordinates, UUID and version, the +typed predicate, selected partitions, empty-pruning and COUNT flags, split size +and time zone. Projection does not change file enumeration. Each scan node still +executes its own predicates and publishes its own metrics; `PlanningReuseHit` +identifies a reused plan. The same behavior applies through the HMS gateway. + +The existing planning scope releases the memo when planning ends, including on +failure. Nothing is shared across statements. Concurrent identical requests share +one load; a cancelled waiter leaves without cancelling its producer. Failed or +cancelled loads are retryable. Values hold immutable ranges and metrics, without +Kernel rows, snapshots, iterators or resources requiring close. + +Per catalog/query, the memo retains at most 128 entries, 16,384 ranges and 16 MiB +of estimated range payload, whichever fills first. The byte estimate includes +paths, partition values and inline deletion vectors, and is not a bound on the +whole query's heap or in-flight planning. Oversized results are returned normally +without retention. Streaming scans never enter the memo or materialize a file +list for reuse. The optimization uses existing SPI and configuration, with no +changes to the shared FE or other connectors. + +## Verification + +The standard FE UT reactor includes Delta and its metastore module. Run +`./run-fe-ut.sh --run 'DeltaLake*Test,DeltaKernel*Test'` for the connector suites, +and the FE/Hive statistics tests for engine integration. The connector tests cover cold lookups, concurrent +single-flight, user/version/UUID/path isolation, missing/invalid statistics, +checksum version matching, deletion vectors, V2 checkpoints, retry, invalidation +and close. Eight concurrent scans share one cached real checkpoint/DV snapshot; +half close iterators early while the other scans must enumerate all files. + +`DeltaLakePlanningScaleTest` is opt-in through `deltalake.planning.table`, using the +retained million-file planning fixture documented in the Delta p2 README. It also +checks asynchronous statistics in a bounded test JVM. + +`DeltaLakeScanPlanMemoTest` and `DeltaLakeScanPlanProviderTest` cover reuse, +isolation, typed predicates, COUNT, descriptor ownership, planning lifetime, +capacity, concurrent loading, failure and cancellation. Run +`run-regression-test.sh --run -d external_table_p0/deltalake +-s test_deltalake_scan_reuse` to compare disabled/enabled synchronous planning and +streaming through filesystem, dedicated HMS and HMS gateway catalogs, including +deletion vectors and column mapping. The existing mixed HMS/Iceberg suite checks +the gateway's parallel format path. + +Run `run-regression-test.sh --run -d external_table_p0/deltalake +-s test_deltalake_statistics` against the optional Delta fixtures. The suite checks +size-based and unknown CBO estimates with collection disabled, and exact COUNT +through filesystem, HMS and Hive gateway catalogs, including after a table refresh. +The optional benchmark checks refreshed estimates against the fixture manifest. diff --git a/fe/fe-connector/fe-connector-deltalake/pom.xml b/fe/fe-connector/fe-connector-deltalake/pom.xml new file mode 100644 index 00000000000000..6d52077819244b --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/pom.xml @@ -0,0 +1,204 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-deltalake + jar + Doris FE Connector - Delta Lake + Native Delta Lake metadata and scan planning with Delta Kernel. + + + + + io.delta + delta-kernel-api + 4.4.1 + + + + io.delta + delta-kernel-defaults + 4.4.1 + + + org.apache.hadoop + hadoop-client-runtime + + + org.apache.hadoop + hadoop-client-api + + + io.unitycatalog + unitycatalog-client + + + io.unitycatalog + unitycatalog-hadoop + + + + + + + ${project.groupId} + fe-connector-spi + ${project.version} + + + + ${project.groupId} + fe-connector-cache + ${project.version} + + + + ${project.groupId} + fe-filesystem-api + ${project.version} + + + + ${project.groupId} + fe-foundation + ${project.version} + + + + ${project.groupId} + fe-connector-hms + ${project.version} + + + + ${project.groupId} + fe-connector-metastore-hms + ${project.version} + + + ${project.groupId} + fe-connector-metastore-deltalake + ${project.version} + + + + ${project.groupId} + fe-thrift + ${project.version} + provided + + + + org.roaringbitmap + RoaringBitmap + 0.9.25 + + + + com.github.ben-manes.caffeine + caffeine + 2.9.3 + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + runtime + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.slf4j + slf4j-reload4j + + + + + + org.apache.hadoop + hadoop-aws + + + + software.amazon.awssdk + s3 + + + + software.amazon.awssdk + apache-client + + + + org.apache.logging.log4j + log4j-api + + + + org.junit.jupiter + junit-jupiter + test + + + + + doris-fe-connector-deltalake + + + maven-assembly-plugin + + false + + src/main/assembly/plugin-zip.xml + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/assembly/plugin-zip.xml b/fe/fe-connector/fe-connector-deltalake/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..bd7b94590ecba0 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/assembly/plugin-zip.xml @@ -0,0 +1,68 @@ + + + + + plugin + + zip + + false + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + src/main/resources/deltalake.conf.template + / + + + + + + /lib + false + runtime + + org.apache.doris:fe-connector-spi + org.apache.doris:fe-extension-spi + org.apache.doris:fe-filesystem-api + org.apache.doris:fe-thrift + org.apache.thrift:libthrift + + org.apache.logging.log4j:* + org.slf4j:* + + + + diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaKernelCompat.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaKernelCompat.java new file mode 100644 index 00000000000000..06d65e8aba6477 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaKernelCompat.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.data.Row; +import io.delta.kernel.internal.InternalScanFileUtils; +import io.delta.kernel.internal.actions.DeletionVectorDescriptor; +import io.delta.kernel.internal.actions.Protocol; +import io.delta.kernel.internal.snapshot.LogSegment; +import io.delta.kernel.internal.tablefeatures.TableFeature; +import io.delta.kernel.internal.util.PartitionUtils; +import io.delta.kernel.statistics.DataFileStatistics; +import io.delta.kernel.types.DataType; +import io.delta.kernel.utils.FileStatus; + +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.stream.Collectors; + +/** The only production class coupled to Kernel internal APIs; pinned to Kernel 4.4.1. */ +final class DeltaKernelCompat { + private DeltaKernelCompat() { + } + + static String tableId(Snapshot snapshot) { + return snapshot.getMetadata().getId(); + } + + static OptionalLong fileCount(Snapshot snapshot) { + return snapshot.getCurrentCrcInfo().map(crc -> OptionalLong.of(crc.getNumFiles())) + .orElseGet(OptionalLong::empty); + } + + static Optional tableStatistics(Snapshot snapshot) { + // A version checksum has no total row count. Only an empty active-file set proves zero rows. + return snapshot.getCurrentCrcInfo().map(crc -> new ConnectorTableStatistics( + crc.getNumFiles() == 0 ? 0 : -1, crc.getTableSizeBytes())); + } + + static void prepareForConcurrentReads(Snapshot snapshot) { + // Kernel's Lazy is not synchronized. Materialize read-side metadata before sharing a snapshot. + snapshot.getSchema(); + snapshot.getTableProperties(); + snapshot.getMetadata().getDataSchema(); + snapshot.getScanBuilder(); + snapshot.getCurrentCrcInfo(); + snapshot.getLogSegment().allFilesWithCompactionsReversed(); + } + + static long snapshotRetainedBytes(Snapshot snapshot) { + prepareForConcurrentReads(snapshot); + LogSegment segment = snapshot.getLogSegment(); + // Kernel 4.4.1 retains metadata/schema copies, lazy lookup maps and log FileStatus lists. + // Charge a conservative expansion of schema JSON and CRC bytes, including optional allFiles + // embedded in a CRC. The catalog's shared Engine/FileIO service graph is not entry-owned. + long bytes = 16384L + 32L * snapshot.getMetadata().getSchemaString().length(); + for (Map.Entry property : snapshot.getTableProperties().entrySet()) { + bytes = JvmSizeUtils.saturatedAdd(bytes, + 128L + 4L * (property.getKey().length() + property.getValue().length())); + } + for (FileStatus file : segment.allLogFilesReversed()) { + bytes = JvmSizeUtils.saturatedAdd(bytes, 256L + 4L * file.getPath().length()); + } + for (FileStatus file : segment.getCompactions()) { + bytes = JvmSizeUtils.saturatedAdd(bytes, 256L + 4L * file.getPath().length()); + } + if (segment.getLastSeenChecksum().isPresent()) { + long size = segment.getLastSeenChecksum().get().getSize(); + bytes = JvmSizeUtils.saturatedAdd(bytes, size > Long.MAX_VALUE / 32 ? Long.MAX_VALUE : size * 32); + } + return bytes; + } + + static FileStatus fileStatus(Row file) { + return InternalScanFileUtils.getAddFileStatus(file); + } + + static Map partitionValues(Row file) { + return InternalScanFileUtils.getPartitionValues(file); + } + + static Object partitionLiteral(DataType type, String value) { + return PartitionUtils.literalForPartitionValue(type, value).getValue(); + } + + static OptionalLong numRecords(Row file) { + Row add = file.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL); + int ordinal = InternalScanFileUtils.ADD_FILE_STATS_ORDINAL; + if (add.isNullAt(ordinal)) { + return OptionalLong.empty(); + } + Optional count = DataFileStatistics.getNumRecords(add.getString(ordinal)); + return count.isPresent() ? OptionalLong.of(count.get()) : OptionalLong.empty(); + } + + static long deletionCardinality(Row file) { + DeletionVectorDescriptor dv = InternalScanFileUtils.getDeletionVectorDescriptorFromRow(file); + return dv == null ? 0 : dv.getCardinality(); + } + + static Optional deletionVector(Row file, String tablePath) { + DeletionVectorDescriptor dv = InternalScanFileUtils.getDeletionVectorDescriptorFromRow(file); + if (dv == null || dv.getCardinality() == 0) { + return Optional.empty(); + } + TDeltaLakeDeletionVectorDesc result = new TDeltaLakeDeletionVectorDesc(); + result.setSizeInBytes(dv.getSizeInBytes()); + result.setCardinality(dv.getCardinality()); + if (dv.isInline()) { + result.setInlineData(dv.inlineData()); + } else { + result.setPath(dv.getAbsolutePath(tablePath)); + result.setOffset(dv.getOffset().orElse(0)); + } + return Optional.of(result); + } + + static Set readerFeatures(Snapshot snapshot) { + return readerFeatures(snapshot.getProtocol()); + } + + static Set readerFeatures(Protocol protocol) { + // Kernel 4.4.1's legacy-protocol branch also selects writer-only features (reader version 0). + // Those do not require reader support and must not reject an otherwise readable table. + return protocol.getImplicitlyAndExplicitlySupportedReaderWriterFeatures().stream() + .filter(feature -> feature.minReaderVersion() > 0) + .map(TableFeature::featureName).collect(Collectors.toSet()); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogProperties.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogProperties.java new file mode 100644 index 00000000000000..f5e676146db691 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogProperties.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.metastore.MetaStoreProperties; +import org.apache.doris.connector.metastore.spi.MetaStoreProviders; +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.ParamRules; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** Catalog-owned choices; metastore and storage properties stay with their respective binders. */ +final class DeltaLakeCatalogProperties { + static final String CATALOG_TYPE = "deltalake.catalog.type"; + static final String FILESYSTEM = "filesystem"; + static final String HMS = "hms"; + + @ConnectorProperty(names = {CATALOG_TYPE}, required = false, + description = "Delta metadata source: filesystem (default) or hms.") + private String catalogType = FILESYSTEM; + + @ConnectorProperty(names = {"enable.mapping.timestamp_tz"}, required = false, + description = "Map Delta timestamps representing instants to TIMESTAMPTZ.") + private boolean mapTimestampTz; + + @ConnectorProperty(names = {"enable.mapping.varbinary"}, required = false, + description = "Map Delta binary columns to VARBINARY instead of STRING.") + private boolean mapVarbinary; + + // Shared HMS client setting, also consumed by Hive and Hudi. + @ConnectorProperty(names = {"hive.metastore.client.pool.size"}, required = false, + description = "Maximum number of pooled Hive Metastore clients.") + private int hmsClientPoolSize = 8; + + private final Map raw; + private MetaStoreProperties metastore; + + private DeltaLakeCatalogProperties(Map properties) { + raw = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + static DeltaLakeCatalogProperties of(Map properties) { + DeltaLakeCatalogProperties result = new DeltaLakeCatalogProperties(properties); + ConnectorPropertiesUtils.bindConnectorProperties(result, properties); + result.catalogType = result.catalogType.toLowerCase(Locale.ROOT); + if (!FILESYSTEM.equals(result.catalogType) && !HMS.equals(result.catalogType)) { + throw new IllegalArgumentException("Unsupported " + CATALOG_TYPE + ": " + result.catalogType + + "; expected filesystem or hms"); + } + result.metastore = MetaStoreProviders.bindForType(result.catalogType, properties, Collections.emptyMap()); + result.metastore.validate(); + DeltaLakeSnapshotCache.validate(properties); + new ParamRules().check(() -> result.hmsClientPoolSize < 0, + "hive.metastore.client.pool.size must not be negative").validate(); + return result; + } + + String getCatalogType() { + return catalogType; + } + + MetaStoreProperties getMetastore() { + return metastore; + } + + boolean isMapTimestampTz() { + return mapTimestampTz; + } + + boolean isMapVarbinary() { + return mapVarbinary; + } + + int getHmsClientPoolSize() { + return hmsClientPoolSize; + } + + Map getRaw() { + return raw; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnector.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnector.java new file mode 100644 index 00000000000000..f989580bc27efc --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnector.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorCapability; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +final class DeltaLakeConnector implements Connector { + private static final Logger LOG = LogManager.getLogger(DeltaLakeConnector.class); + private static final ExecutorService RESOURCE_CLOSER = resourceCloser(); + + private final DeltaLakeCatalogProperties properties; + private final ConnectorContext context; + private final CatalogMetaCache metaCache; + private final DeltaLakeSnapshotCache snapshotCache; + private final DeltaLakeStatisticsCache statisticsCache; + private DeltaLakeRuntime runtime; + private boolean closed; + + DeltaLakeConnector(Map properties, ConnectorContext context) { + this.properties = DeltaLakeCatalogProperties.of(properties); + this.context = context; + metaCache = CatalogMetaCache.managed(context.getCatalogId(), "deltalake", properties); + snapshotCache = new DeltaLakeSnapshotCache(metaCache, properties); + statisticsCache = new DeltaLakeStatisticsCache(metaCache, properties); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return new DeltaLakeConnectorMetadata(runtime(), properties, snapshotCache, statisticsCache); + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return new DeltaLakeScanPlanProvider(runtime(), statisticsCache); + } + + private synchronized DeltaLakeRuntime runtime() { + if (closed) { + throw new DorisConnectorException("Delta catalog is closed: " + context.getCatalogName()); + } + if (runtime == null) { + // Resource construction sets up local configuration and lazy clients; network reads start + // after this monitor has been released, in metadata and scan operations. + runtime = new DeltaLakeRuntime(properties, context); + } + return runtime; + } + + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof DeltaLakeTableHandle; + } + + @Override + public Set getCapabilities() { + return Collections.singleton(ConnectorCapability.SUPPORTS_MVCC_SNAPSHOT); + } + + @Override + public OptionalLong schemaCacheTtlSecondOverride() { + // Schema is derived from the statement's Delta snapshot, never a separately cached latest schema. + return OptionalLong.of(0); + } + + @Override + public void invalidateTable(String database, String table) { + metaCache.invalidateTable(database, table); + } + + @Override + public void invalidateDb(String database) { + metaCache.invalidateDatabase(database); + } + + @Override + public void invalidateAll() { + metaCache.invalidateCatalog(); + } + + @Override + public void close() { + DeltaLakeRuntime owned; + synchronized (this) { + closed = true; + owned = runtime; + runtime = null; + } + metaCache.close(); + if (owned != null) { + // ALTER/DROP may hold the catalog write lock. S3/HDFS client shutdown can wait for + // networking threads; perform it outside that lock, as the Hudi connector does. + RESOURCE_CLOSER.execute(() -> { + try { + owned.close(); + } catch (IOException e) { + LOG.warn("Failed to close Delta catalog resources for {}", context.getCatalogName(), e); + } + }); + } + } + + private static ExecutorService resourceCloser() { + ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), action -> { + Thread thread = new Thread(action, "deltalake-resource-closer"); + thread.setDaemon(true); + thread.setContextClassLoader(DeltaLakeConnector.class.getClassLoader()); + return thread; + }); + executor.allowCoreThreadTimeOut(true); + return executor; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorMetadata.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorMetadata.java new file mode 100644 index 00000000000000..9548144d035fc3 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorMetadata.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.handle.NamedColumnHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.mvcc.ConnectorTimeTravelSpec; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import io.delta.kernel.Snapshot; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +final class DeltaLakeConnectorMetadata implements ConnectorMetadata { + private static final String TABLE_ID = "deltalake.table-id"; + private final DeltaLakeRuntime runtime; + private final DeltaLakeCatalogProperties properties; + private final DeltaLakeSnapshotCache snapshots; + private final DeltaLakeStatisticsCache statistics; + + DeltaLakeConnectorMetadata(DeltaLakeRuntime runtime, DeltaLakeCatalogProperties properties, + DeltaLakeSnapshotCache snapshots, DeltaLakeStatisticsCache statistics) { + this.runtime = runtime; + this.properties = properties; + this.snapshots = snapshots; + this.statistics = statistics; + } + + @Override + public List listDatabaseNames(ConnectorSession session) { + return runtime.execute(runtime.resolver::listDatabases); + } + + @Override + public boolean databaseExists(ConnectorSession session, String database) { + return listDatabaseNames(session).contains(database); + } + + @Override + public List listTableNames(ConnectorSession session, String database) { + return runtime.execute(() -> runtime.resolver.listTables(database)); + } + + @Override + public Optional getTableHandle(ConnectorSession session, String database, String table) { + return DeltaLakeStatementScope.resolve(session, database, table, () -> runtime.execute(() -> { + Optional location = runtime.resolver.tableLocation(database, table); + if (location.isEmpty()) { + return Optional.empty(); + } + Snapshot snapshot = snapshots.latest(session, database, table, location.get(), runtime.engine); + return Optional.of(new DeltaLakeTableHandle(database, table, snapshot)); + })).map(handle -> handle); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + DeltaLakeTableHandle delta = (DeltaLakeTableHandle) handle; + return runtime.execute(() -> DeltaLakeSchemaBuilder.build(delta.getTable(), + delta.snapshot(runtime.engine), properties)); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + return getTableSchema(session, applySnapshot(session, handle, snapshot)); + } + + @Override + public Optional getTableStatistics( + ConnectorSession session, ConnectorTableHandle handle) { + return runtime.execute(() -> statistics.get(session, (DeltaLakeTableHandle) handle, runtime)); + } + + @Override + public Optional getTableStatistics(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + return getTableStatistics(session, applySnapshot(session, handle, snapshot)); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle handle) { + Map columns = new LinkedHashMap<>(); + for (ConnectorColumn column : getTableSchema(session, handle).getColumns()) { + columns.put(column.getName(), new NamedColumnHandle(column.getName())); + } + return columns; + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + return getColumnHandles(session, applySnapshot(session, handle, snapshot)); + } + + @Override + public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) { + return true; + } + + @Override + public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + return listPartitions(session, handle, Optional.empty()).stream() + .map(ConnectorPartitionInfo::getPartitionName).collect(Collectors.toList()); + } + + @Override + public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, + Optional filter) { + return runtime.execute(() -> { + Snapshot snapshot = ((DeltaLakeTableHandle) handle).snapshot(runtime.engine); + if (snapshot.getPartitionColumnNames().isEmpty()) { + return Collections.emptyList(); + } + Map partitions = new LinkedHashMap<>(); + long[] files = {0}; + DeltaLakeScanFiles.forEach(snapshot, runtime.engine, false, row -> { + files[0]++; + Map values = DeltaLakePartitionValues.normalize( + snapshot, DeltaKernelCompat.partitionValues(row)); + String name = DeltaLakePartitionValues.partitionName(values); + partitions.computeIfAbsent(name, ignored -> new ConnectorPartitionInfo(name, values, + Collections.emptyMap(), values.values().stream().map(value -> value == null ? "" : value) + .collect(Collectors.toList()), + values.values().stream().map(value -> value == null).collect(Collectors.toList()))); + }); + DeltaLakeStatementScope.fileCount(session, (DeltaLakeTableHandle) handle).value = files[0]; + return List.copyOf(partitions.values()); + }); + } + + @Override + public boolean listsPartitionsAtSnapshot(ConnectorSession session, ConnectorTableHandle handle) { + return true; + } + + @Override + public Optional beginQuerySnapshot(ConnectorSession session, ConnectorTableHandle handle) { + DeltaLakeTableHandle delta = (DeltaLakeTableHandle) handle; + return Optional.of(ConnectorMvccSnapshot.builder().snapshotId(delta.getVersion()) + .schemaId(delta.getVersion()).retainSchema(true) + .properties(Collections.singletonMap(TABLE_ID, delta.getTableId())).build()); + } + + @Override + public Optional resolveTimeTravel(ConnectorSession session, ConnectorTableHandle handle, + ConnectorTimeTravelSpec spec) { + if (spec.getKind() == ConnectorTimeTravelSpec.Kind.VERSION_REF) { + throw invalidVersion(spec.getStringValue()); + } + if (spec.getKind() != ConnectorTimeTravelSpec.Kind.SNAPSHOT_ID) { + return Optional.empty(); + } + long version; + try { + version = Long.parseLong(spec.getStringValue()); + } catch (NumberFormatException e) { + throw invalidVersion(spec.getStringValue()); + } + if (version < 0) { + throw invalidVersion(spec.getStringValue()); + } + // Even a matching latest pointer may be cached across statements. Open the explicit target + // once in this statement so a recreated table or truncated version cannot use that pointer. + return beginQuerySnapshot(session, loadVersion(session, (DeltaLakeTableHandle) handle, version)); + } + + private static DorisConnectorException invalidVersion(String value) { + return new DorisConnectorException("Delta Lake FOR VERSION AS OF requires a non-negative 64-bit integer: " + + value); + } + + @Override + public ConnectorTableHandle applySnapshot(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + DeltaLakeTableHandle delta = (DeltaLakeTableHandle) handle; + if (!delta.getTableId().equals(snapshot.getProperties().get(TABLE_ID))) { + throw new DorisConnectorException("Delta snapshot belongs to another table: " + delta.getPath()); + } + return delta.getVersion() == snapshot.getSnapshotId() + ? delta : loadVersion(session, delta, snapshot.getSnapshotId()); + } + + private DeltaLakeTableHandle loadVersion(ConnectorSession session, DeltaLakeTableHandle delta, long version) { + return DeltaLakeStatementScope.atVersion(session, delta, version, + () -> runtime.execute(() -> delta.atVersion(runtime.engine, version))); + } + + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, long tableId, + String tableName, String database, String remoteName, int columns, long catalogId) { + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, columns, 0, tableName, database); + descriptor.setHiveTable(new THiveTable(database, tableName, Collections.emptyMap())); + return descriptor; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorProvider.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorProvider.java new file mode 100644 index 00000000000000..dac8f8d2cc6da8 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorProvider.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import java.util.Map; + +public final class DeltaLakeConnectorProvider implements ConnectorProvider { + @Override + public String getType() { + return "deltalake"; + } + + @Override + public String displayEngineName() { + return "Delta Lake"; + } + + @Override + public void validateProperties(Map properties) { + DeltaLakeCatalogProperties.of(properties); + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + return new DeltaLakeConnector(properties, context); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheck.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheck.java new file mode 100644 index 00000000000000..473131857228c3 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheck.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TypeChange; + +import java.util.Set; +import java.util.TreeSet; + +/** Fail before planning files when a table requires semantics the native reader has not validated. */ +final class DeltaLakeFeatureCheck { + // Features are enabled alongside their end-to-end read coverage. + static final Set SUPPORTED_READER_FEATURES = Set.of( + "columnMapping", "timestampNtz", "typeWidening", "typeWidening-preview", "deletionVectors", + "v2Checkpoint", "vacuumProtocolCheck"); + + private DeltaLakeFeatureCheck() { + } + + static void check(Snapshot snapshot) { + check(snapshot.getPath(), DeltaKernelCompat.readerFeatures(snapshot)); + checkWidening(snapshot.getSchema(), snapshot.getPath()); + } + + static void check(String table, Set readerFeatures) { + Set unsupported = new TreeSet<>(readerFeatures); + unsupported.removeAll(SUPPORTED_READER_FEATURES); + if (!unsupported.isEmpty()) { + throw new DorisConnectorException("Unsupported Delta Lake reader features " + unsupported + + " for table " + table); + } + } + + static void checkWidening(StructType schema, String path) { + for (StructField field : schema.fields()) { + checkFieldWidening(field, path + "." + field.getName()); + } + } + + private static void checkFieldWidening(StructField field, String path) { + for (TypeChange change : field.getTypeChanges()) { + if (!supportedWidening(change.getFrom(), change.getTo())) { + throw new DorisConnectorException("Unsupported Delta Lake type widening for " + path + + ": " + change.getFrom() + " -> " + change.getTo()); + } + } + DataType type = field.getDataType(); + if (type instanceof StructType) { + checkWidening((StructType) type, path); + } else if (type instanceof ArrayType) { + checkFieldWidening(((ArrayType) type).getElementField(), path + ".element"); + } else if (type instanceof MapType) { + checkFieldWidening(((MapType) type).getKeyField(), path + ".key"); + checkFieldWidening(((MapType) type).getValueField(), path + ".value"); + } + } + + static boolean supportedWidening(DataType from, DataType to) { + int fromWidth = integerWidth(from); + int toWidth = integerWidth(to); + if (fromWidth > 0 && toWidth > fromWidth) { + return true; + } + if (fromWidth > 0) { + if (fromWidth <= 32 && to instanceof DoubleType) { + return true; + } + if (to instanceof DecimalType) { + DecimalType decimal = (DecimalType) to; + int digits = fromWidth == 8 ? 3 : fromWidth == 16 ? 5 : fromWidth == 32 ? 10 : 19; + return decimal.getPrecision() - decimal.getScale() >= digits; + } + } + if (from instanceof FloatType && to instanceof DoubleType) { + return true; + } + if (from instanceof DateType && to instanceof TimestampNTZType) { + return true; + } + if (from instanceof DecimalType && to instanceof DecimalType) { + DecimalType oldType = (DecimalType) from; + DecimalType newType = (DecimalType) to; + return newType.getScale() >= oldType.getScale() + && newType.getPrecision() - newType.getScale() >= oldType.getPrecision() - oldType.getScale(); + } + return false; + } + + private static int integerWidth(DataType type) { + if (type instanceof ByteType) { + return 8; + } + if (type instanceof ShortType) { + return 16; + } + if (type instanceof IntegerType) { + return 32; + } + return type instanceof LongType ? 64 : 0; + } + +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileIO.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileIO.java new file mode 100644 index 00000000000000..b11f0cbd422c56 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileIO.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorContext; + +import com.google.common.util.concurrent.Uninterruptibles; +import io.delta.kernel.defaults.engine.fileio.FileIO; +import io.delta.kernel.defaults.engine.fileio.InputFile; +import io.delta.kernel.defaults.engine.fileio.OutputFile; +import io.delta.kernel.defaults.engine.hadoopio.HadoopInputFile; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.FileStatus; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; + +/** + * Read-only DefaultEngine IO with catalog-owned Hadoop clients. + * + *

Kernel's default HadoopFileIO obtains a filesystem for each operation but never closes it. + * With fs.hdfs.impl.disable.cache=true that creates a new client on each read. Instead, retain one + * private newInstance per authority and close exactly these clients, without closing another catalog's + * global Hadoop cache entries. Directory discovery borrows the same instances. + */ +final class DeltaLakeFileIO implements FileIO, AutoCloseable { + private final Configuration configuration; + private final ConnectorContext context; + // Publish the task under this monitor, then initialize outside it: different authorities and + // catalog close must not wait for a network connection while holding this lock. + private final Map> fileSystems = new HashMap<>(); + private boolean closed; + + DeltaLakeFileIO(Configuration configuration, ConnectorContext context) { + this.configuration = new Configuration(configuration); + this.configuration.setClassLoader(DeltaLakeFileIO.class.getClassLoader()); + this.context = context; + } + + FileSystem fileSystem(Path path) throws IOException { + URI uri = path.makeQualified(FileSystem.getDefaultUri(configuration), new Path("/")).toUri(); + String key = uri.getScheme() + "://" + uri.getAuthority(); + FutureTask task; + synchronized (fileSystems) { + if (closed) { + throw new IOException("Delta catalog filesystem is closed"); + } + task = fileSystems.computeIfAbsent(key, ignored -> new FutureTask<>( + () -> context.executeAuthenticated(() -> FileSystem.newInstance(uri, configuration)))); + } + // FutureTask runs its callable once, including when several readers reach the same authority. + task.run(); + try { + return task.get(); + } catch (InterruptedException e) { + // A waiting query may be cancelled while another query initializes the shared client. + // Keep that initialization cached for its other readers; only this wait is cancelled. + Thread.currentThread().interrupt(); + throw new IOException("Interrupted opening Delta filesystem for " + path, e); + } catch (ExecutionException e) { + synchronized (fileSystems) { + fileSystems.remove(key, task); + } + throw new IOException("Failed to open Delta filesystem for " + path, e.getCause()); + } + } + + @Override + public CloseableIterator listFrom(String filePath) throws IOException { + Path path = new Path(filePath); + org.apache.hadoop.fs.FileStatus[] entries = fileSystem(path).listStatus(path.getParent()); + Arrays.sort(entries, Comparator.comparing(entry -> entry.getPath().getName())); + Iterator files = Arrays.stream(entries) + .filter(entry -> entry.isFile() && entry.getPath().getName().compareTo(path.getName()) >= 0) + .map(DeltaLakeFileIO::status) + .iterator(); + return new CloseableIterator() { + @Override + public boolean hasNext() { + return files.hasNext(); + } + + @Override + public FileStatus next() { + return files.next(); + } + + @Override + public void close() { + // Hadoop listStatus has already released its listing resources. + } + }; + } + + @Override + public FileStatus getFileStatus(String path) throws IOException { + Path file = new Path(path); + return status(fileSystem(file).getFileStatus(file)); + } + + private static FileStatus status(org.apache.hadoop.fs.FileStatus file) { + return FileStatus.of(file.getPath().toString(), file.getLen(), file.getModificationTime()); + } + + @Override + public String resolvePath(String path) throws IOException { + Path file = new Path(path); + return fileSystem(file).makeQualified(file).toString(); + } + + @Override + public InputFile newInputFile(String path, long fileSize) { + Path file = new Path(path); + try { + return new HadoopInputFile(fileSystem(file), file, fileSize); + } catch (IOException e) { + throw new UncheckedIOException("Failed to open Delta input " + path, e); + } + } + + @Override + public Optional getConf(String key) { + return Optional.ofNullable(configuration.get(key)); + } + + @Override + public boolean mkdirs(String path) { + throw new UnsupportedOperationException("Delta Lake connector is read-only"); + } + + @Override + public OutputFile newOutputFile(String path) { + throw new UnsupportedOperationException("Delta Lake connector is read-only"); + } + + @Override + public boolean delete(String path) { + throw new UnsupportedOperationException("Delta Lake connector is read-only"); + } + + @Override + public void copyFileAtomically(String source, String destination, boolean overwrite) { + throw new UnsupportedOperationException("Delta Lake connector is read-only"); + } + + @Override + public void close() throws IOException { + List> owned; + synchronized (fileSystems) { + closed = true; + owned = new ArrayList<>(fileSystems.values()); + fileSystems.clear(); + } + IOException failure = null; + for (FutureTask task : owned) { + try { + Uninterruptibles.getUninterruptibly(task).close(); + } catch (ExecutionException e) { + // Initialization failed and yielded no filesystem; the reading thread reports it. + } catch (IOException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolver.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolver.java new file mode 100644 index 00000000000000..c6f6a77277a4c6 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolver.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** Filesystem catalogs expose exactly warehouse/database/table/_delta_log. */ +final class DeltaLakeFileSystemTableResolver implements DeltaLakeTableResolver { + private final Path warehouse; + private final DeltaLakeFileIO fileIO; + + DeltaLakeFileSystemTableResolver(String warehouse, DeltaLakeFileIO fileIO) { + this.warehouse = new Path(warehouse); + this.fileIO = fileIO; + } + + @Override + public List listDatabases() throws IOException { + List databases = new ArrayList<>(); + for (FileStatus child : children(warehouse)) { + if (child.isDirectory()) { + databases.add(child.getPath().getName()); + } + } + Collections.sort(databases); + return databases; + } + + @Override + public List listTables(String database) throws IOException { + Path databasePath = child(warehouse, database); + List tables = new ArrayList<>(); + for (FileStatus entry : children(databasePath)) { + if (entry.isDirectory() && hasDeltaLog(entry.getPath())) { + tables.add(entry.getPath().getName()); + } + } + Collections.sort(tables); + return tables; + } + + @Override + public Optional tableLocation(String database, String table) throws IOException { + Path path = child(child(warehouse, database), table); + return hasDeltaLog(path) ? Optional.of(fileIO.resolvePath(path.toString())) + : Optional.empty(); + } + + private FileStatus[] children(Path path) throws IOException { + try { + return fileIO.fileSystem(path).listStatus(path); + } catch (FileNotFoundException e) { + // A new warehouse or an absent database is an empty namespace, not an IO failure. + return new FileStatus[0]; + } + } + + private boolean hasDeltaLog(Path table) throws IOException { + Path log = new Path(table, "_delta_log"); + FileSystem fileSystem = fileIO.fileSystem(log); + try { + return fileSystem.getFileStatus(log).isDirectory(); + } catch (FileNotFoundException e) { + return false; + } + } + + private static Path child(Path parent, String name) { + // Hadoop Path treats an absolute child or URI as a replacement for the warehouse. + if (name.isEmpty() || ".".equals(name) || "..".equals(name) + || name.indexOf('/') >= 0 || name.indexOf('\\') >= 0 || name.indexOf(':') >= 0) { + throw new IllegalArgumentException("Delta filesystem database/table must be a single path name: " + name); + } + return new Path(parent, name); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolver.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolver.java new file mode 100644 index 00000000000000..45fa0339ec9a48 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolver.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.spi.DorisConnectorException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** HMS provides discovery and location only, never Delta schema or partition metadata. */ +final class DeltaLakeHmsTableResolver implements DeltaLakeTableResolver { + private final HmsClient client; + + DeltaLakeHmsTableResolver(HmsClient client) { + this.client = client; + } + + @Override + public List listDatabases() { + return client.listDatabases(); + } + + @Override + public List listTables(String database) { + List tables = new ArrayList<>(); + for (String table : client.listTables(database)) { + if (isDelta(client.getTable(database, table))) { + tables.add(table); + } + } + return tables; + } + + @Override + public Optional tableLocation(String database, String table) { + if (!client.tableExists(database, table)) { + return Optional.empty(); + } + HmsTableInfo info = client.getTable(database, table); + if (!isDelta(info)) { + return Optional.empty(); + } + // Spark's data-source tables can put a dummy location ending in -__PLACEHOLDER__ in + // StorageDescriptor. The SerDe path is the authoritative Delta root in that representation. + String location = info.getSdParameters().getOrDefault("path", info.getLocation()); + if (location == null || location.isEmpty()) { + throw new DorisConnectorException("Delta table " + database + "." + table + " has no HMS location"); + } + return Optional.of(location); + } + + static boolean isDelta(HmsTableInfo table) { + return "delta".equalsIgnoreCase(table.getParameters().get("spark.sql.sources.provider")); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValues.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValues.java new file mode 100644 index 00000000000000..060f73b00a3997 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValues.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.scan.ConnectorPartitionValues; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; + +import java.math.BigDecimal; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** Log partition values, in declaration order, with genuine nulls kept distinct from string values. */ +final class DeltaLakePartitionValues { + private static final String PHYSICAL_NAME = "delta.columnMapping.physicalName"; + + private DeltaLakePartitionValues() { + } + + static String partitionName(Map values) { + return values.entrySet().stream().map(entry -> escape(entry.getKey()) + "=" + + (entry.getValue() == null ? ConnectorPartitionValues.NULL_PARTITION_NAME + : escapeValue(entry.getValue()))).collect(Collectors.joining("/")); + } + + private static String escapeValue(String value) { + // A real string equal to the null marker must keep a different partition identity. + return ConnectorPartitionValues.NULL_PARTITION_NAME.equals(value) + ? "%5F" + value.substring(1) : escape(value); + } + + private static String escape(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20"); + } + + static Map normalize(Snapshot snapshot, Map rawValues) { + boolean mapped = !"none".equals(snapshot.getTableProperties().getOrDefault("delta.columnMapping.mode", "none")); + return normalize(snapshot.getSchema(), snapshot.getPartitionColumnNames(), mapped, rawValues); + } + + static Map normalize(StructType schema, List partitions, + boolean mapped, Map rawValues) { + Map result = new LinkedHashMap<>(); + for (String name : partitions) { + StructField field = schema.get(name); + String key = mapped ? field.getMetadata().getString(PHYSICAL_NAME) : name; + if (!rawValues.containsKey(key)) { + throw new DorisConnectorException("Delta file has no partition value for column " + name); + } + try { + result.put(name, normalizeValue(field.getDataType(), rawValues.get(key))); + } catch (IllegalArgumentException | DateTimeException e) { + throw new DorisConnectorException("Invalid Delta partition value for column " + name + + " of type " + field.getDataType(), e); + } + } + return Collections.unmodifiableMap(result); + } + + static String normalizeValue(DataType type, String value) { + if (value == null || type instanceof StringType || type instanceof BinaryType) { + // Delta serializes binary partition values as UTF-8 strings, not base64. Do not + // path-decode either type: percent signs, slashes, empty strings and sentinels are data. + return value; + } + if (type instanceof TimestampType) { + // The protocol's space-separated timestamps also mean UTC. java.sql.Timestamp's + // JVM-default-zone parsing can normalize a DST gap, so use java.time directly. + TemporalAccessor parsed = DateTimeFormatter.ISO_DATE_TIME.parseBest(value.replace(' ', 'T'), + OffsetDateTime::from, LocalDateTime::from); + OffsetDateTime instant = parsed instanceof OffsetDateTime + ? ((OffsetDateTime) parsed).withOffsetSameInstant(ZoneOffset.UTC) + : ((LocalDateTime) parsed).atOffset(ZoneOffset.UTC); + return instant.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } + if (type instanceof TimestampNTZType) { + return LocalDateTime.parse(value.replace(' ', 'T'), DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + } + if (type instanceof DateType) { + return LocalDate.parse(value).toString(); + } + if (type instanceof BooleanType && !"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw new IllegalArgumentException("Expected a boolean partition value"); + } + Object literal = DeltaKernelCompat.partitionLiteral(type, value); + return literal instanceof BigDecimal ? ((BigDecimal) literal).toPlainString() : literal.toString(); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverter.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverter.java new file mode 100644 index 00000000000000..cb33fcfe0d4af9 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverter.java @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.pushdown.ConnectorAnd; +import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.spi.pushdown.ConnectorComparison; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.pushdown.ConnectorIn; +import org.apache.doris.connector.spi.pushdown.ConnectorIsNull; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.pushdown.ConnectorNot; +import org.apache.doris.connector.spi.pushdown.ConnectorOr; + +import io.delta.kernel.expressions.Column; +import io.delta.kernel.expressions.Expression; +import io.delta.kernel.expressions.In; +import io.delta.kernel.expressions.Literal; +import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.types.BasePrimitiveType; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** File pruning only: every original predicate remains a BE residual. */ +final class DeltaLakePredicateConverter { + private final StructType schema; + private final ZoneId zone; + + DeltaLakePredicateConverter(StructType schema, ZoneId zone) { + this.schema = schema; + this.zone = zone; + } + + Optional convert(ConnectorExpression expression) { + if (expression instanceof ConnectorAnd) { + Predicate result = null; + for (ConnectorExpression child : expression.getChildren()) { + Optional converted = convert(child); + if (converted.isPresent()) { + result = result == null ? converted.get() : new Predicate("AND", result, converted.get()); + } + } + return Optional.ofNullable(result); + } + return Optional.ofNullable(exact(expression)); + } + + private Predicate exact(ConnectorExpression expression) { + if (expression instanceof ConnectorAnd || expression instanceof ConnectorOr) { + Predicate result = null; + for (ConnectorExpression child : expression.getChildren()) { + Predicate converted = exact(child); + if (converted == null) { + // Dropping a child beneath OR or NOT can exclude matching rows. Only top-level + // ANDs in convert() may retain a subset of their conjuncts. + return null; + } + result = result == null ? converted : new Predicate( + expression instanceof ConnectorAnd ? "AND" : "OR", result, converted); + } + return result; + } + if (expression instanceof ConnectorNot) { + Predicate child = exact(((ConnectorNot) expression).getOperand()); + return child == null ? null : new Predicate("NOT", child); + } + if (expression instanceof ConnectorIsNull) { + ConnectorIsNull isNull = (ConnectorIsNull) expression; + StructField field = field(isNull.getOperand()); + return field == null ? null : new Predicate(isNull.isNegated() ? "IS_NOT_NULL" : "IS_NULL", + new Column(field.getName())); + } + if (expression instanceof ConnectorIn) { + ConnectorIn in = (ConnectorIn) expression; + StructField field = field(in.getValue()); + if (field == null || in.getInList().isEmpty()) { + return null; + } + List values = new ArrayList<>(); + for (ConnectorExpression item : in.getInList()) { + Literal value = literal(field.getDataType(), item); + if (value == null) { + return null; + } + values.add(value); + } + Predicate predicate = new In(new Column(field.getName()), values); + return in.isNegated() ? new Predicate("NOT", predicate) : predicate; + } + if (expression instanceof ConnectorComparison) { + ConnectorComparison comparison = (ConnectorComparison) expression; + boolean reversed = comparison.getLeft() instanceof ConnectorLiteral; + StructField field = field(reversed ? comparison.getRight() : comparison.getLeft()); + if (field == null) { + return null; + } + Literal value = literal(field.getDataType(), reversed ? comparison.getLeft() : comparison.getRight()); + if (value == null) { + return null; + } + Column column = new Column(field.getName()); + String operator = comparison.getOperator().getSymbol(); + if (comparison.getOperator() == ConnectorComparison.Operator.EQ_FOR_NULL) { + // Keep null-safe semantics even beneath NOT; '=' would turn null into UNKNOWN. + operator = "IS NOT DISTINCT FROM"; + } + if (comparison.getOperator() == ConnectorComparison.Operator.NE) { + return new Predicate("NOT", new Predicate("=", column, value)); + } + return reversed ? new Predicate(operator, value, column) : new Predicate(operator, column, value); + } + return null; + } + + private StructField field(ConnectorExpression expression) { + if (!(expression instanceof ConnectorColumnRef)) { + return null; + } + String name = ((ConnectorColumnRef) expression).getColumnName(); + for (StructField field : schema.fields()) { + if (field.getName().equalsIgnoreCase(name)) { + DataType type = field.getDataType(); + // Kernel 4.4.1 parses min/max using the current type. Old DATE statistics + // contain only yyyy-MM-dd and fail its timestamp_ntz parser after widening. + // Keep predicates on these columns as BE residuals until Kernel handles them. + if (field.getTypeChanges().stream().anyMatch(change -> change.getFrom() instanceof DateType + && change.getTo() instanceof TimestampNTZType)) { + return null; + } + // The SPI currently carries root-column references, not nested field paths. + return (type instanceof BasePrimitiveType || type instanceof DecimalType) ? field : null; + } + } + return null; + } + + Literal literal(DataType type, ConnectorExpression expression) { + if (!(expression instanceof ConnectorLiteral) + || !(type instanceof BasePrimitiveType || type instanceof DecimalType)) { + return null; + } + ConnectorLiteral literal = (ConnectorLiteral) expression; + Object value = literal.getValue(); + if (literal.isNull()) { + return Literal.ofNull(type); + } + try { + if (type instanceof BooleanType && value instanceof Boolean) { + return Literal.ofBoolean((Boolean) value); + } + if (type instanceof StringType && ((StringType) type).isUTF8BinaryCollated() + && value instanceof String) { + return Literal.ofString((String) value); + } + if (type instanceof BinaryType && value instanceof byte[]) { + return Literal.ofBinary((byte[]) value); + } + if (type instanceof DateType && value instanceof LocalDate) { + return Literal.ofDate(Math.toIntExact(((LocalDate) value).toEpochDay())); + } + if ((type instanceof TimestampType || type instanceof TimestampNTZType) + && value instanceof LocalDateTime) { + LocalDateTime dateTime = (LocalDateTime) value; + boolean adjusted = type instanceof TimestampType; + boolean utcCarrier = "TIMESTAMPTZ".equalsIgnoreCase(literal.getType().getTypeName()); + Instant instant = dateTime.atZone(adjusted && !utcCarrier ? zone : ZoneOffset.UTC).toInstant(); + long micros = Math.addExact(Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), + instant.getNano() / 1000L); + return adjusted ? Literal.ofTimestamp(micros) : Literal.ofTimestampNtz(micros); + } + if (value instanceof Number) { + if (type instanceof FloatType && "FLOAT".equalsIgnoreCase(literal.getType().getTypeName())) { + float number = ((Number) value).floatValue(); + return Float.isFinite(number) ? Literal.ofFloat(number) : null; + } + if (type instanceof DoubleType && (value instanceof Double || value instanceof Float)) { + double number = ((Number) value).doubleValue(); + return Double.isFinite(number) ? Literal.ofDouble(number) : null; + } + if (value instanceof Double || value instanceof Float) { + // Comparing an exact integer/decimal column in floating-point space can match + // multiple adjacent exact values. An integral-looking float is still lossy. + return null; + } + BigDecimal number = new BigDecimal(value.toString()); + if (type instanceof DecimalType) { + DecimalType decimal = (DecimalType) type; + BigDecimal scaled = number.setScale(decimal.getScale(), RoundingMode.UNNECESSARY); + return scaled.precision() <= decimal.getPrecision() + ? Literal.ofDecimal(scaled, decimal.getPrecision(), decimal.getScale()) : null; + } + if (type instanceof ByteType) { + return Literal.ofByte(number.byteValueExact()); + } + if (type instanceof ShortType) { + return Literal.ofShort(number.shortValueExact()); + } + if (type instanceof IntegerType) { + return Literal.ofInt(number.intValueExact()); + } + if (type instanceof LongType) { + return Literal.ofLong(number.longValueExact()); + } + } + } catch (ArithmeticException | NumberFormatException e) { + // Out-of-range or inexact coercions remain residuals, never rounded pruning boundaries. + return null; + } + return null; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRowCount.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRowCount.java new file mode 100644 index 00000000000000..3a632f44aa757b --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRowCount.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.data.Row; + +import java.util.OptionalLong; + +/** Logical rows in a complete, unfiltered snapshot; shared by CBO statistics and exact COUNT pushdown. */ +final class DeltaLakeRowCount { + private long count; + private boolean complete = true; + + void add(Row file) { + OptionalLong records = DeltaKernelCompat.numRecords(file); + long deleted = DeltaKernelCompat.deletionCardinality(file); + if (records.isEmpty()) { + complete = false; + } else { + if (deleted < 0 || records.getAsLong() < deleted) { + throw new DorisConnectorException("Invalid Delta row count or deletion cardinality for " + + DeltaKernelCompat.fileStatus(file).getPath()); + } + count = Math.addExact(count, records.getAsLong() - deleted); + } + } + + OptionalLong value() { + return complete ? OptionalLong.of(count) : OptionalLong.empty(); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRuntime.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRuntime.java new file mode 100644 index 00000000000000..6b6d8f5b7e401b --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeRuntime.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsClientConfig; +import org.apache.doris.connector.hms.HmsConfHelper; +import org.apache.doris.connector.hms.ThriftHmsClient; +import org.apache.doris.connector.metastore.FileSystemMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.AbstractHmsMetaStoreProperties; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.filesystem.properties.StorageProperties; +import org.apache.doris.kerberos.AuthType; +import org.apache.doris.kerberos.AuthenticationConfig; +import org.apache.doris.kerberos.HadoopAuthenticator; +import org.apache.doris.kerberos.KerberosAuthSpec; +import org.apache.doris.kerberos.KerberosAuthenticationConfig; + +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; + +/** Lazily built plugin resources; constructing catalog properties never opens storage or HMS. */ +final class DeltaLakeRuntime implements AutoCloseable { + final Engine engine; + final DeltaLakeTableResolver resolver; + final ConnectorContext storageContext; + private final DeltaLakeFileIO fileIO; + private final HmsClient hms; + + DeltaLakeRuntime(DeltaLakeCatalogProperties properties, ConnectorContext context) { + HmsConfHelper.initializeHadoopConfigDir(context); + Configuration configuration = storageConfiguration(properties.getRaw(), context); + HadoopAuthenticator storageAuth = storageAuthenticator(configuration); + storageContext = new TcclPinningConnectorContext(context, () -> storageAuth); + fileIO = new DeltaLakeFileIO(configuration, storageContext); + engine = DefaultEngine.create(fileIO); + if (DeltaLakeCatalogProperties.HMS.equals(properties.getCatalogType())) { + AbstractHmsMetaStoreProperties metastore = (AbstractHmsMetaStoreProperties) properties.getMetastore(); + HadoopAuthenticator hmsAuth = hmsAuthenticator(metastore); + ConnectorContext hmsContext = new TcclPinningConnectorContext(context, () -> hmsAuth); + HmsClientConfig config = new HmsClientConfig(metastore.getConfResources(), + HmsConfHelper.mergeCatalogProperties(properties.getRaw(), + metastore.toHiveConfOverrides(HmsConfHelper.metastoreClientTimeoutSecond(context))), + properties.getHmsClientPoolSize()); + hms = new ThriftHmsClient(config, new ThriftHmsClient.AuthAction() { + @Override + public T execute(Callable action) throws Exception { + return hmsContext.executeAuthenticated(action); + } + }); + resolver = new DeltaLakeHmsTableResolver(hms); + } else { + hms = null; + resolver = new DeltaLakeFileSystemTableResolver( + ((FileSystemMetaStoreProperties) properties.getMetastore()).getWarehouse(), fileIO); + } + } + + static Configuration storageConfiguration(Map properties, ConnectorContext context) { + Map merged = new HashMap<>(); + for (StorageProperties storage : context.getStorageContext().getStorageProperties()) { + storage.toHadoopProperties().ifPresent(value -> merged.putAll(value.toHadoopConfigurationMap())); + } + // Keep the same precedence as Paimon: normalized storage settings first, explicit Hadoop keys last. + properties.forEach((key, value) -> { + if (key.startsWith("fs.") || key.startsWith("dfs.") || key.startsWith("hadoop.")) { + merged.put(key, value); + } + }); + Configuration configuration = HmsConfHelper.createHadoopConfWithResources( + merged.get("hadoop.config.resources"), merged); + configuration.setClassLoader(DeltaLakeRuntime.class.getClassLoader()); + return configuration; + } + + static HadoopAuthenticator storageAuthenticator(Configuration configuration) { + // The plugin owns its Hadoop UGI copy. Explicit SIMPLE users need the same treatment as + // Kerberos principals; otherwise an app-loader doAs leaves HDFS using the FE process user. + boolean kerberos = "kerberos".equalsIgnoreCase(configuration.get("hadoop.security.authentication")); + boolean explicitUser = !configuration.getTrimmed("hadoop.username", "").isEmpty(); + return kerberos || explicitUser ? HadoopAuthenticator.getHadoopAuthenticator(configuration) : null; + } + + static HadoopAuthenticator hmsAuthenticator(AbstractHmsMetaStoreProperties properties) { + Configuration configuration = HmsConfHelper.createHadoopConfWithResources( + properties.getConfResources(), properties.toHiveConfOverrides("")); + Optional kerberos = properties.kerberos(); + if (kerberos.isPresent() && kerberos.get().hasCredentials()) { + KerberosAuthSpec spec = kerberos.get(); + configuration.set("hadoop.security.authentication", "kerberos"); + configuration.set("hive.metastore.sasl.enabled", "true"); + return HadoopAuthenticator.getHadoopAuthenticator( + new KerberosAuthenticationConfig(spec.getPrincipal(), spec.getKeytab(), configuration)); + } + if (properties.getAuthType() == AuthType.KERBEROS) { + return null; + } + // HMS SIMPLE user selection belongs to the metastore, not to the storage identity. + return HadoopAuthenticator.getHadoopAuthenticator( + AuthenticationConfig.getSimpleAuthenticationConfig(configuration)); + } + + T execute(Callable action) { + try { + return storageContext.executeAuthenticated(action); + } catch (RuntimeException e) { + throw e; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DorisConnectorException("Delta metadata operation interrupted for catalog '" + + storageContext.getCatalogName() + "'", e); + } catch (Exception e) { + throw new DorisConnectorException("Delta metadata operation failed for catalog '" + + storageContext.getCatalogName() + "'", e); + } + } + + @Override + public void close() throws IOException { + // A null HMS client is valid for a filesystem catalog. Try-with-resources also preserves + // both failures if closing the filesystem and the HMS pool each reports an error. + try (HmsClient ownedHms = hms) { + fileIO.close(); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanFiles.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanFiles.java new file mode 100644 index 00000000000000..cd5ee0a5511e4e --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanFiles.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.ScanBuilder; +import io.delta.kernel.Snapshot; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.data.Row; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.utils.CloseableIterator; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.function.Consumer; + +/** Consumers must copy values they retain: Kernel rows belong to the current batch. */ +final class DeltaLakeScanFiles { + private DeltaLakeScanFiles() { + } + + static void forEach(Snapshot snapshot, Engine engine, boolean includeStats, Consumer consumer) + throws IOException { + forEach(snapshot, engine, includeStats, Optional.empty(), consumer); + } + + static void forEach(Snapshot snapshot, Engine engine, boolean includeStats, Optional filter, + Consumer consumer) throws IOException { + try (CloseableIterator files = open(snapshot, engine, includeStats, filter)) { + while (files.hasNext()) { + consumer.accept(files.next()); + } + } + } + + static CloseableIterator open(Snapshot snapshot, Engine engine, boolean includeStats, + Optional filter) { + return new Files(() -> { + ScanBuilder builder = snapshot.getScanBuilder(); + filter.ifPresent(builder::withFilter); + return builder.build().getScanFiles(engine, includeStats); + }); + } + + interface Opener { + CloseableIterator open() throws IOException; + } + + static final class Files implements CloseableIterator { + private final Opener opener; + private CloseableIterator batches; + private CloseableIterator rows; + private boolean closed; + + Files(Opener opener) { + this.opener = opener; + } + + @Override + public boolean hasNext() { + if (closed) { + return false; + } + try { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedIOException("Delta file planning interrupted"); + } + if (batches == null) { + batches = opener.open(); + } + while (rows == null || !rows.hasNext()) { + closeRows(); + if (!batches.hasNext()) { + close(); + return false; + } + rows = batches.next().getRows(); + } + return true; + } catch (IOException e) { + closeAfterFailure(e); + throw new DorisConnectorException("Failed to iterate Delta scan files", e); + } catch (RuntimeException | Error e) { + closeAfterFailure(e); + throw e; + } + } + + @Override + public Row next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + try { + return rows.next(); + } catch (RuntimeException | Error e) { + closeAfterFailure(e); + throw e; + } + } + + private void closeRows() throws IOException { + CloseableIterator previous = rows; + rows = null; + if (previous != null) { + previous.close(); + } + } + + private void closeAfterFailure(Throwable error) { + try { + close(); + } catch (IOException | RuntimeException e) { + error.addSuppressed(e); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + try (CloseableIterator owned = batches) { + closeRows(); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanMetrics.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanMetrics.java new file mode 100644 index 00000000000000..bcbbe2fb1cbcff --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanMetrics.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.scan.ConnectorScanProfile; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.engine.ExpressionHandler; +import io.delta.kernel.engine.FileSystemClient; +import io.delta.kernel.engine.JsonHandler; +import io.delta.kernel.engine.MetricsReporter; +import io.delta.kernel.engine.ParquetHandler; +import io.delta.kernel.metrics.ScanReport; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.LongConsumer; + +/** One producer owns this collector. Only the immutable result is published to the explain thread. */ +final class DeltaLakeScanMetrics { + private final DeltaLakeTableHandle handle; + private final DeltaLakeStatementScope.FileCount totalFiles; + private final LongConsumer fileCountObserver; + private final Set> partitions = new HashSet<>(); + private long selectedFiles; + private long splits; + private long countPartitions; + private boolean complete; + + DeltaLakeScanMetrics(ConnectorSession session, DeltaLakeTableHandle handle, Snapshot snapshot, + LongConsumer fileCountObserver) { + this.handle = handle; + this.fileCountObserver = fileCountObserver; + totalFiles = DeltaLakeStatementScope.fileCount(session, handle); + DeltaKernelCompat.fileCount(snapshot).ifPresent(value -> totalFiles.value = value); + } + + Engine engine(Engine delegate) { + List reporters = new ArrayList<>(delegate.getMetricsReporters()); + reporters.add(report -> { + if (report instanceof ScanReport) { + ScanReport scan = (ScanReport) report; + // Partition predicates can prune checkpoint rows before the active-file counter. + // Only an exhaustive scan without that predicate establishes the table total. + if (scan.getIsFullyConsumed() && scan.getPartitionPredicate().isEmpty()) { + totalFiles.value = scan.getScanMetrics().getNumActiveAddFiles(); + fileCountObserver.accept(totalFiles.value); + } + } + }); + return new Engine() { + @Override + public ExpressionHandler getExpressionHandler() { + return delegate.getExpressionHandler(); + } + + @Override + public JsonHandler getJsonHandler() { + return delegate.getJsonHandler(); + } + + @Override + public FileSystemClient getFileSystemClient() { + return delegate.getFileSystemClient(); + } + + @Override + public ParquetHandler getParquetHandler() { + return delegate.getParquetHandler(); + } + + @Override + public List getMetricsReporters() { + return reporters; + } + }; + } + + void file(Map values) { + selectedFiles++; + if (!values.isEmpty()) { + partitions.add(values); + } + } + + void split() { + splits++; + } + + void count(long files, long partitionCount) { + selectedFiles = files; + totalFiles.value = files; + fileCountObserver.accept(files); + countPartitions = partitionCount; + } + + void complete() { + complete = true; + } + + Result result() { + return new Result(handle.withoutSnapshot(), totalFiles.value, selectedFiles, splits, + countPartitions + partitions.size(), complete); + } + + static final class Result { + private final DeltaLakeTableHandle handle; + private final long total; + private final long selected; + private final long splits; + private final long partitions; + private final boolean complete; + + private Result(DeltaLakeTableHandle handle, long total, long selected, long splits, + long partitions, boolean complete) { + this.handle = handle; + this.total = total; + this.selected = selected; + this.splits = splits; + this.partitions = partitions; + this.complete = complete; + } + + void explain(StringBuilder output, String prefix) { + output.append(prefix).append("Delta files: ").append(selected).append('/') + .append(total < 0 ? "unknown" : Long.toString(total)) + .append(", selected partitions: ").append(partitions) + .append(", splits: ").append(splits) + .append(complete ? "" : " (partial)").append('\n'); + } + + ConnectorScanProfile profile(boolean planningReuseHit) { + Map metrics = new LinkedHashMap<>(); + metrics.put("SnapshotVersion", Long.toString(handle.getVersion())); + metrics.put("TotalFiles", total < 0 ? "unknown" : Long.toString(total)); + metrics.put("SelectedFiles", Long.toString(selected)); + metrics.put("SelectedPartitions", Long.toString(partitions)); + metrics.put("GeneratedSplits", Long.toString(splits)); + metrics.put("EnumerationComplete", Boolean.toString(complete)); + metrics.put("PlanningReuseHit", Boolean.toString(planningReuseHit)); + return new ConnectorScanProfile("Delta Scan Metrics", + "Table Scan (" + handle.getDatabase() + "." + handle.getTable() + ")", metrics); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemo.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemo.java new file mode 100644 index 00000000000000..fdcb5b1a23bb19 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemo.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.scan.ConnectorScanContext; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +/** Planning-only values: no snapshots, Kernel rows, iterators, sessions or resources to close. */ +final class DeltaLakeScanPlanMemo { + static final String NAMESPACE = "deltalake.scan-plans"; + private final int maxEntries; + private final int maxRanges; + private final long maxRangeBytes; + private final Map> plans = new HashMap<>(); + private int retainedRanges; + private long retainedRangeBytes; + + DeltaLakeScanPlanMemo() { + this(128, 16_384, 16L << 20); + } + + DeltaLakeScanPlanMemo(int maxEntries, int maxRanges, long maxRangeBytes) { + this.maxEntries = maxEntries; + this.maxRanges = maxRanges; + this.maxRangeBytes = maxRangeBytes; + } + + Outcome get(Key key, ConnectorScanContext context, Supplier loader) { + checkCancelled(context); + CompletableFuture future; + boolean owner = false; + synchronized (this) { + future = plans.get(key); + if (future == null && plans.size() < maxEntries) { + future = new CompletableFuture<>(); + plans.put(key, future); + owner = true; + } + } + if (future == null) { + Plan plan = loader.get(); + checkCancelled(context); + return new Outcome(plan, false); + } + if (!owner) { + return new Outcome(await(future, context), true); + } + // Never load under the scope's computeIfAbsent or this memo's monitor: loading also + // resolves statement file counts and COUNT summaries, and unrelated scans may run in parallel. + try { + Plan plan = loader.get(); + checkCancelled(context); + long bytes = rangeBytes(plan); + synchronized (this) { + if (plan.ranges.size() <= maxRanges - retainedRanges + && bytes <= maxRangeBytes - retainedRangeBytes) { + retainedRanges += plan.ranges.size(); + retainedRangeBytes += bytes; + } else { + plans.remove(key); + } + } + future.complete(plan); + return new Outcome(plan, false); + } catch (RuntimeException | Error failure) { + synchronized (this) { + plans.remove(key); + } + future.completeExceptionally(failure); + throw failure; + } + } + + private long rangeBytes(Plan plan) { + if (plan.ranges.size() > maxRanges) { + return Long.MAX_VALUE; + } + long bytes = 64L + 8L * plan.ranges.size(); + for (ConnectorScanRange range : plan.ranges) { + bytes += ((DeltaLakeScanRange) range).estimatedRetainedBytes(); + if (bytes > maxRangeBytes) { + return bytes; + } + } + return bytes; + } + + private static Plan await(CompletableFuture future, ConnectorScanContext context) { + while (true) { + checkCancelled(context); + try { + Plan plan = future.get(50, TimeUnit.MILLISECONDS); + checkCancelled(context); + return plan; + } catch (TimeoutException ignored) { + // A cancelled waiter must leave without cancelling the shared producer. + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new CancellationException("Delta scan planning interrupted"); + } catch (ExecutionException failure) { + if (failure.getCause() instanceof RuntimeException) { + throw (RuntimeException) failure.getCause(); + } + throw (Error) failure.getCause(); + } + } + } + + static void checkCancelled(ConnectorScanContext context) { + context.checkCancelled(); + if (Thread.currentThread().isInterrupted()) { + throw new CancellationException("Delta scan planning interrupted"); + } + } + + // Projection, LIMIT, aggregate/explain flags and backend capabilities do not affect Delta's + // file enumeration. Schema/column mapping and DV descriptors are fixed by table ID + version. + record Key(String user, String database, String table, String path, String tableId, long version, + Optional filter, Set partitions, boolean prunedToEmpty, + boolean countPushdown, long splitSize, String timeZone) { + static Key of(ConnectorSession session, ConnectorScanRequest request, long splitSize) { + DeltaLakeTableHandle table = (DeltaLakeTableHandle) request.getTableHandle(); + return new Key(session.getUser(), table.getDatabase(), table.getTable(), table.getPath(), + table.getTableId(), table.getVersion(), request.getFilter(), + Set.copyOf(request.getRequiredPartitions()), request.isPartitionsPrunedToEmpty(), + request.isCountPushdown(), splitSize, session.getTimeZone()); + } + } + + record Plan(List ranges, DeltaLakeScanMetrics.Result metrics) { + Plan { + // The loader owns the list and never mutates it after publication. Avoid copying an + // oversized synchronous plan just to find that it cannot be retained. + ranges = Collections.unmodifiableList(ranges); + } + } + + record Outcome(Plan plan, boolean reused) { + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProvider.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProvider.java new file mode 100644 index 00000000000000..90c32322b8061f --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProvider.java @@ -0,0 +1,479 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorPushdownOps; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScopes; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.scan.ConnectorScanContext; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanProfile; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.scan.ConnectorSplitSource; +import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TDeltaLakeFileDesc; +import org.apache.doris.thrift.TFileScanRangeParams; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.data.Row; +import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.FileStatus; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +final class DeltaLakeScanPlanProvider implements ConnectorScanPlanProvider, ConnectorPushdownOps { + private static final String SCHEMA_MAPPING = "deltalake.schema_mapping"; + private static final long DEFAULT_SPLIT_SIZE = 128L << 20; + private final DeltaLakeRuntime runtime; + private final DeltaLakeStatisticsCache statistics; + // The connector creates a provider per scan node. The gate runs before that node opens its source. + private boolean streamingCount; + private volatile DeltaLakeScanMetrics.Result scanMetrics; + private boolean planningReuseHit; + + DeltaLakeScanPlanProvider(DeltaLakeRuntime runtime, DeltaLakeStatisticsCache statistics) { + this.runtime = runtime; + this.statistics = statistics; + } + + @Override + public boolean supportsCastPredicatePushdown(ConnectorSession session) { + // CAST(123.45 AS BIGINT) matches 123. The unwrapped decimal predicate would skip its file. + return false; + } + + @Override + public long streamingSplitEstimate(ConnectorSession session, ConnectorTableHandle handle, + Optional filter, boolean countPushdown) { + streamingCount = false; + String enabled = session.getProperty("enable_external_table_batch_mode", String.class); + if (enabled != null && !Boolean.parseBoolean(enabled)) { + return -1; + } + String configured = session.getProperty("num_files_in_batch_mode", String.class); + long threshold = configured == null ? 10000 : Long.parseLong(configured); + return runtime.execute(() -> { + DeltaLakeTableHandle delta = (DeltaLakeTableHandle) handle; + Snapshot snapshot = delta.snapshot(runtime.engine); + OptionalLong files = DeltaKernelCompat.fileCount(snapshot); + if (files.isEmpty()) { + long observed = DeltaLakeStatementScope.fileCount(session, delta).value; + files = observed >= 0 ? OptionalLong.of(observed) : statistics.cachedFileCount(session, delta); + } + // Old writers may not produce a checksum. Unknown size streams conservatively instead + // of enumerating AddFiles or waiting for statistics merely to choose a scheduling mode. + if (files.isPresent() && files.getAsLong() < threshold) { + return -1L; + } + streamingCount = countPushdown && filter.isEmpty(); + return files.orElse(threshold); + }); + } + + @Override + public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorScanRequest request) { + planningReuseHit = false; + request.getScanContext().checkCancelled(); + return runtime.execute(() -> new StreamingSource(session, request)); + } + + private final class StreamingSource implements ConnectorSplitSource { + private final ConnectorSession session; + private final DeltaLakeTableHandle handle; + private final Snapshot snapshot; + private final Optional filter; + private final long targetSize; + private final boolean mapped; + private final boolean countPushdown; + private final ConnectorScanContext scanContext; + private final Set requiredPartitions; + private final boolean partitionsPrunedToEmpty; + private final DeltaLakeScanMetrics metrics; + private CloseableIterator files; + private DeltaLakeScanRange file; + private DeltaLakeScanRange buffered; + private long offset; + private boolean countChecked; + private boolean countOnly; + private boolean closed; + + private StreamingSource(ConnectorSession session, ConnectorScanRequest request) { + this.session = session; + handle = (DeltaLakeTableHandle) request.getTableHandle(); + snapshot = handle.snapshot(runtime.engine); + DeltaLakeFeatureCheck.check(snapshot); + filter = predicate(session, snapshot, request.getFilter()); + targetSize = targetSplitSize(session); + mapped = DeltaLakeSchemaBuilder.hasColumnMapping(snapshot); + scanContext = request.getScanContext(); + requiredPartitions = new HashSet<>(request.getRequiredPartitions()); + partitionsPrunedToEmpty = request.isPartitionsPrunedToEmpty(); + countPushdown = streamingCount && request.getFilter().isEmpty() && requiredPartitions.isEmpty(); + metrics = new DeltaLakeScanMetrics(session, handle, snapshot, + statistics.fileCountObserver(session, handle, runtime)); + scanMetrics = null; + } + + @Override + public boolean hasNext() { + if (closed) { + return false; + } + try { + scanContext.checkCancelled(); + if (Thread.currentThread().isInterrupted()) { + throw new DorisConnectorException("Delta split planning interrupted for " + handle.getPath()); + } + if (buffered != null) { + return true; + } + return runtime.execute(() -> { + if (partitionsPrunedToEmpty) { + metrics.complete(); + close(); + return false; + } + if (countPushdown && !countChecked) { + countChecked = true; + CountSummary summary = countSummary(session, handle); + countOnly = summary.count.isPresent(); + if (countOnly) { + metrics.count(summary.files, summary.partitions); + } + if (countOnly && summary.first != null) { + buffered = summary.first.withRowCount(summary.count.getAsLong()); + return true; + } + } + if (countOnly) { + metrics.complete(); + close(); + return false; + } + if (files == null) { + files = DeltaLakeScanFiles.open(snapshot, metrics.engine(runtime.engine), false, filter); + } + while (file == null || offset == file.getFileSize()) { + scanContext.checkCancelled(); + if (!files.hasNext()) { + metrics.complete(); + close(); + return false; + } + Row row = files.next(); + Map partitions = DeltaLakePartitionValues.normalize( + snapshot, DeltaKernelCompat.partitionValues(row)); + if (!requiredPartitions.isEmpty() && !requiredPartitions.contains( + DeltaLakePartitionValues.partitionName(partitions))) { + continue; + } + file = fileRange(row, handle, partitions, targetSize, mapped); + metrics.file(file.getPartitionValues()); + offset = 0; + } + long length = Math.min(targetSize, file.getFileSize() - offset); + buffered = file.withByteRange(offset, length); + offset += length; + return true; + }); + } catch (RuntimeException | Error e) { + try { + close(); + } catch (RuntimeException closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + } + + @Override + public ConnectorScanRange next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + ConnectorScanRange result = buffered; + buffered = null; + metrics.split(); + return result; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + buffered = null; + file = null; + try { + if (files != null) { + runtime.execute(() -> { + files.close(); + return null; + }); + } + } finally { + scanMetrics = metrics.result(); + } + } + } + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + scanMetrics = null; + planningReuseHit = false; + return runtime.execute(() -> { + DeltaLakeScanPlanMemo.checkCancelled(request.getScanContext()); + DeltaLakeTableHandle handle = (DeltaLakeTableHandle) request.getTableHandle(); + Snapshot snapshot = handle.snapshot(runtime.engine); + DeltaLakeFeatureCheck.check(snapshot); + if (session == null || !session.isExternalScanTaskReuseEnabled()) { + return planScanInternal(session, request, handle, snapshot).ranges(); + } + String namespace = DeltaLakeScanPlanMemo.NAMESPACE + ":" + session.getCatalogId() + + ":" + session.getQueryId(); + DeltaLakeScanPlanMemo memo = session.getStatementScope().computeIfAbsentForPlanning( + namespace, DeltaLakeScanPlanMemo::new); + DeltaLakeScanPlanMemo.Outcome outcome = memo.get( + DeltaLakeScanPlanMemo.Key.of(session, request, targetSplitSize(session)), request.getScanContext(), + () -> planScanInternal(session, request, handle, snapshot)); + planningReuseHit = outcome.reused(); + scanMetrics = outcome.plan().metrics(); + return outcome.plan().ranges(); + }); + } + + private DeltaLakeScanPlanMemo.Plan planScanInternal(ConnectorSession session, ConnectorScanRequest request, + DeltaLakeTableHandle handle, Snapshot snapshot) { + DeltaLakeScanMetrics metrics = new DeltaLakeScanMetrics(session, handle, snapshot, + statistics.fileCountObserver(session, handle, runtime)); + try { + List ranges = planRanges(session, request, handle, snapshot, metrics); + DeltaLakeScanPlanMemo.checkCancelled(request.getScanContext()); + metrics.complete(); + return new DeltaLakeScanPlanMemo.Plan(ranges, metrics.result()); + } catch (IOException e) { + throw new DorisConnectorException("Failed to plan Delta scan files for " + handle.getPath(), e); + } finally { + scanMetrics = metrics.result(); + } + } + + private List planRanges(ConnectorSession session, ConnectorScanRequest request, + DeltaLakeTableHandle handle, Snapshot snapshot, DeltaLakeScanMetrics metrics) throws IOException { + if (request.isPartitionsPrunedToEmpty()) { + return Collections.emptyList(); + } + long targetSize = targetSplitSize(session); + boolean mapped = DeltaLakeSchemaBuilder.hasColumnMapping(snapshot); + if (request.isCountPushdown() && request.getFilter().isEmpty() + && request.getRequiredPartitions().isEmpty()) { + CountSummary summary = countSummary(session, handle); + if (summary.count.isPresent()) { + metrics.count(summary.files, summary.partitions); + if (summary.first == null) { + return Collections.emptyList(); + } + metrics.split(); + return Collections.singletonList(summary.first.withRowCount(summary.count.getAsLong())); + } + } + Set required = new HashSet<>(request.getRequiredPartitions()); + List ranges = new ArrayList<>(); + Optional predicate = predicate(session, snapshot, request.getFilter()); + DeltaLakeScanFiles.forEach(snapshot, metrics.engine(runtime.engine), false, predicate, row -> { + DeltaLakeScanPlanMemo.checkCancelled(request.getScanContext()); + Map partitions = DeltaLakePartitionValues.normalize( + snapshot, DeltaKernelCompat.partitionValues(row)); + if (!required.isEmpty() && !required.contains(DeltaLakePartitionValues.partitionName(partitions))) { + return; + } + DeltaLakeScanRange file = fileRange(row, handle, partitions, targetSize, mapped); + metrics.file(partitions); + for (long offset = 0; offset < file.getFileSize();) { + DeltaLakeScanPlanMemo.checkCancelled(request.getScanContext()); + long length = Math.min(targetSize, file.getFileSize() - offset); + ranges.add(file.withByteRange(offset, length)); + metrics.split(); + offset += length; + } + }); + return ranges; + } + + private DeltaLakeScanRange fileRange(Row row, DeltaLakeTableHandle handle, + Map partitions, long targetSize, boolean mapped) { + FileStatus file = DeltaKernelCompat.fileStatus(row); + if (file.getSize() <= 0) { + throw new DorisConnectorException("Invalid Delta Parquet file size for " + file.getPath()); + } + String path = runtime.storageContext.getStorageContext().normalizeStorageUri(file.getPath()); + TDeltaLakeFileDesc descriptor = new TDeltaLakeFileDesc(); + if (mapped) { + descriptor.setSchemaId(DeltaLakeSchemaBuilder.FILE_SCHEMA_ID); + } + DeltaKernelCompat.deletionVector(row, handle.getPath()).ifPresent(dv -> { + if (dv.isSetPath()) { + dv.setPath(runtime.storageContext.getStorageContext().normalizeStorageUri(dv.getPath())); + } + descriptor.setDeletionVector(dv); + }); + return new DeltaLakeScanRange(path, 0, file.getSize(), file.getSize(), file.getModificationTime(), + targetSize, partitions, descriptor); + } + + private static Optional predicate(ConnectorSession session, Snapshot snapshot, + Optional filter) { + return filter.flatMap(expression -> new DeltaLakePredicateConverter(snapshot.getSchema(), + ZoneId.of(session.getTimeZone())).convert(expression)); + } + + @Override + public boolean canServeMetadataOnlyCount(ConnectorSession session, ConnectorTableHandle handle, + Optional filter) { + return filter.isEmpty() && runtime.execute(() -> countSummary(session, (DeltaLakeTableHandle) handle) + .count.isPresent()); + } + + private CountSummary countSummary(ConnectorSession session, DeltaLakeTableHandle handle) { + DeltaLakeStatementScope.FileCount observedCount = DeltaLakeStatementScope.fileCount(session, handle); + return ConnectorStatementScopes.resolveInStatement(session, "deltalake.count", handle.getDatabase(), + handle.getTable() + ":" + handle.getTableId() + ":" + handle.getVersion(), () -> runtime.execute(() -> { + Snapshot snapshot = handle.snapshot(runtime.engine); + DeltaLakeFeatureCheck.check(snapshot); + boolean partitioned = !snapshot.getPartitionColumnNames().isEmpty(); + DeltaLakeRowCount count = new DeltaLakeRowCount(); + long[] fileCount = {0}; + Set> partitions = new HashSet<>(); + DeltaLakeScanRange[] first = {null}; + DeltaLakeScanFiles.forEach(snapshot, runtime.engine, true, row -> { + fileCount[0]++; + Map values = partitioned ? DeltaLakePartitionValues.normalize( + snapshot, DeltaKernelCompat.partitionValues(row)) : Collections.emptyMap(); + if (!values.isEmpty()) { + partitions.add(values); + } + if (first[0] == null) { + first[0] = fileRange(row, handle, values, targetSplitSize(session), + DeltaLakeSchemaBuilder.hasColumnMapping(snapshot)); + } + count.add(row); + }); + observedCount.value = fileCount[0]; + return new CountSummary(count.value(), first[0], + fileCount[0], partitions.size()); + })); + } + + private static final class CountSummary { + private final OptionalLong count; + private final DeltaLakeScanRange first; + private final long files; + private final long partitions; + + private CountSummary(OptionalLong count, DeltaLakeScanRange first, long files, long partitions) { + this.count = count; + this.first = first; + this.files = files; + this.partitions = partitions; + } + } + + @Override + public Map getScanNodeProperties(ConnectorSession session, ConnectorTableHandle handle, + List columns, Optional filter) { + return runtime.execute(() -> { + Snapshot snapshot = ((DeltaLakeTableHandle) handle).snapshot(runtime.engine); + Map properties = new HashMap<>(); + properties.put(ScanNodePropertyKeys.FILE_FORMAT_TYPE, "parquet"); + properties.put("deltalake.snapshot_version", String.valueOf(snapshot.getVersion())); + long files = DeltaKernelCompat.fileCount(snapshot).orElse( + DeltaLakeStatementScope.fileCount(session, (DeltaLakeTableHandle) handle).value); + properties.put("deltalake.total_files", files < 0 ? "unknown" : Long.toString(files)); + if (DeltaLakeSchemaBuilder.hasColumnMapping(snapshot)) { + properties.put(SCHEMA_MAPPING, DeltaLakeSchemaBuilder.encodeMapping(snapshot.getSchema())); + } + properties.put(ScanNodePropertyKeys.REQUIRED_CURRENT_BACKEND_SEMANTICS, "Native Delta Lake reads"); + properties.put(ScanNodePropertyKeys.PATH_PARTITION_KEYS, + String.join(",", snapshot.getPartitionColumnNames())); + runtime.storageContext.getStorageContext().getBackendStorageProperties() + .forEach((key, value) -> properties.put(ScanNodePropertyKeys.LOCATION_PREFIX + key, value)); + return properties; + }); + } + + @Override + public void populateScanLevelParams(TFileScanRangeParams params, Map properties) { + DeltaLakeSchemaBuilder.applyMapping(params, properties.get(SCHEMA_MAPPING)); + } + + @Override + public boolean supportsFileCache() { + return true; + } + + @Override + public void appendExplainInfo(StringBuilder output, String prefix, Map properties) { + output.append(prefix).append("Delta snapshot: ").append(properties.get("deltalake.snapshot_version")) + .append('\n'); + DeltaLakeScanMetrics.Result result = scanMetrics; + if (result == null) { + // Streaming EXPLAIN must not eagerly enumerate the lazy source merely for diagnostics. + output.append(prefix).append("Delta files: pending/") + .append(properties.get("deltalake.total_files")) + .append(", selected partitions: pending (streaming)\n"); + } else { + result.explain(output, prefix); + } + } + + @Override + public List collectScanProfiles(ConnectorSession session) { + DeltaLakeScanMetrics.Result result = scanMetrics; + return result == null ? Collections.emptyList() : Collections.singletonList(result.profile(planningReuseHit)); + } + + @Override + public OptionalLong scannedPartitionCount(List ranges) { + return OptionalLong.of(ranges.stream().map(ConnectorScanRange::getPartitionValues) + .filter(values -> !values.isEmpty()).distinct().count()); + } + + private static long targetSplitSize(ConnectorSession session) { + String value = session == null ? null : session.getProperty("file_split_size", String.class); + long configured = value == null || value.isEmpty() ? 0 : Long.parseLong(value); + return configured > 0 ? configured : DEFAULT_SPLIT_SIZE; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanRange.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanRange.java new file mode 100644 index 00000000000000..18ae8f73762226 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeScanRange.java @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; +import org.apache.doris.thrift.TDeltaLakeFileDesc; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** A native Parquet byte range. Partition values and deletion vectors describe the whole data file. */ +final class DeltaLakeScanRange implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + private final String path; + private final long start; + private final long length; + private final long fileSize; + private final long modificationTime; + private final long targetSize; + private final Map partitions; + private final TDeltaLakeFileDesc descriptor; + private final long rowCount; + + DeltaLakeScanRange(String path, long start, long length, long fileSize, long modificationTime, + long targetSize, Map partitions, TDeltaLakeFileDesc descriptor) { + this(path, start, length, fileSize, modificationTime, targetSize, partitions, descriptor, -1); + } + + private DeltaLakeScanRange(String path, long start, long length, long fileSize, long modificationTime, + long targetSize, Map partitions, TDeltaLakeFileDesc descriptor, long rowCount) { + this.path = path; + this.start = start; + this.length = length; + this.fileSize = fileSize; + this.modificationTime = modificationTime; + this.targetSize = targetSize; + this.partitions = Collections.unmodifiableMap(new LinkedHashMap<>(partitions)); + this.descriptor = descriptor.deepCopy(); + this.rowCount = rowCount; + } + + DeltaLakeScanRange withByteRange(long offset, long size) { + if (rowCount == -1 && start == offset && length == size) { + return this; + } + return new DeltaLakeScanRange(path, offset, size, fileSize, modificationTime, targetSize, + partitions, descriptor, -1); + } + + DeltaLakeScanRange withRowCount(long count) { + return new DeltaLakeScanRange(path, start, length, fileSize, modificationTime, targetSize, + partitions, descriptor, count); + } + + // Conservative range-payload estimate for the planning memo. Count shared strings per range; + // the separate entry/range limits also bound empty plans and heavily split single files. + long estimatedRetainedBytes() { + long bytes = 256L + JvmSizeUtils.stringSize(path); + for (Map.Entry partition : partitions.entrySet()) { + bytes += 64L + JvmSizeUtils.stringSize(partition.getKey()) + + JvmSizeUtils.stringSize(partition.getValue()); + } + if (descriptor.isSetDeletionVector()) { + TDeltaLakeDeletionVectorDesc dv = descriptor.getDeletionVector(); + bytes += 128L + JvmSizeUtils.stringSize(dv.getPath()); + if (dv.isSetInlineData()) { + bytes += JvmSizeUtils.byteArraySize(dv.bufferForInlineData().remaining()); + } + } + return bytes; + } + + @Override + public long getPushDownRowCount() { + return rowCount; + } + + @Override + public boolean isNativeReadRange() { + return true; + } + + @Override + public Optional getPath() { + return Optional.of(path); + } + + @Override + public long getStart() { + return start; + } + + @Override + public long getLength() { + return length; + } + + @Override + public long getFileSize() { + return fileSize; + } + + @Override + public long getModificationTime() { + return modificationTime; + } + + @Override + public long getSelfSplitWeight() { + return length; + } + + @Override + public long getTargetSplitSize() { + return targetSize; + } + + @Override + public String getFileFormat() { + return "parquet"; + } + + @Override + public String getTableFormatType() { + return "deltalake"; + } + + @Override + public Map getPartitionValues() { + return partitions; + } + + @Override + public boolean isPartitionBearing() { + // Delta log values are authoritative, including for a table with no partition columns. + return true; + } + + @Override + public Map getProperties() { + return Collections.emptyMap(); + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setDeltalakeParams(descriptor.deepCopy()); + formatDesc.setTableLevelRowCount(rowCount); + rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET); + List keys = new ArrayList<>(partitions.size()); + List values = new ArrayList<>(partitions.size()); + List nulls = new ArrayList<>(partitions.size()); + partitions.forEach((key, value) -> { + keys.add(key); + values.add(value == null ? "" : value); + nulls.add(value == null); + }); + // Override generic Hive path decoding, preserving literal null-sentinel strings as strings. + rangeDesc.setColumnsFromPathKeys(keys); + rangeDesc.setColumnsFromPath(values); + rangeDesc.setColumnsFromPathIsNull(nulls); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilder.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilder.java new file mode 100644 index 00000000000000..038c2d16f699dc --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilder.java @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.thrift.TColumnType; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TPrimitiveType; +import org.apache.doris.thrift.schema.external.TArrayField; +import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; +import org.apache.doris.thrift.schema.external.TMapField; +import org.apache.doris.thrift.schema.external.TNestedField; +import org.apache.doris.thrift.schema.external.TSchema; +import org.apache.doris.thrift.schema.external.TStructField; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.FieldMetadata; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +final class DeltaLakeSchemaBuilder { + static final long TABLE_SCHEMA_ID = 0; + static final long FILE_SCHEMA_ID = 1; + private static final String COLUMN_ID = "delta.columnMapping.id"; + private static final String PHYSICAL_NAME = "delta.columnMapping.physicalName"; + private static final String NESTED_IDS = "delta.columnMapping.nested.ids"; + + private DeltaLakeSchemaBuilder() { + } + + static ConnectorTableSchema build(String table, Snapshot snapshot, DeltaLakeCatalogProperties properties) { + List columns = new ArrayList<>(); + for (StructField field : snapshot.getSchema().fields()) { + String comment = field.getMetadata().contains("comment") ? field.getMetadata().getString("comment") : ""; + // Root names stay remote names, matching partition keys and column handles. The engine + // owns root-name mapping; only nested field names are normalized by the type mapper. + ConnectorColumn column = new ConnectorColumn(field.getName(), + DeltaLakeTypeMapping.toConnectorType(field.getDataType(), + properties.isMapTimestampTz(), properties.isMapVarbinary()), + comment, field.isNullable(), null); + if (field.getDataType() instanceof TimestampType) { + column = column.withTimeZone(); + } + columns.add(column); + } + Map tableProperties = new HashMap<>(snapshot.getTableProperties()); + tableProperties.put(ConnectorTableSchema.SHOW_LOCATION_KEY, snapshot.getPath()); + tableProperties.put(ConnectorTableSchema.PARTITION_COLUMNS_KEY, + String.join(",", snapshot.getPartitionColumnNames())); + return new ConnectorTableSchema(table, columns, "deltalake", tableProperties); + } + + static boolean hasColumnMapping(Snapshot snapshot) { + return !"none".equals(snapshot.getTableProperties().getOrDefault("delta.columnMapping.mode", "none")); + } + + static String encodeMapping(StructType schema) { + TFileScanRangeParams carrier = new TFileScanRangeParams(); + carrier.setCurrentSchemaId(TABLE_SCHEMA_ID); + carrier.setHistorySchemaInfo(List.of(mappingSchema(schema, false), mappingSchema(schema, true))); + try { + byte[] bytes = new TSerializer(new TBinaryProtocol.Factory()).serialize(carrier); + return Base64.getEncoder().encodeToString(bytes); + } catch (TException e) { + throw new DorisConnectorException("Cannot serialize Delta column mapping", e); + } + } + + static void applyMapping(TFileScanRangeParams params, String encoded) { + if (encoded == null) { + // The none mode sends no schema dictionary and uses the reader's BY_NAME path. + return; + } + TFileScanRangeParams carrier = new TFileScanRangeParams(); + try { + new TDeserializer(new TBinaryProtocol.Factory()).deserialize(carrier, Base64.getDecoder().decode(encoded)); + } catch (TException | IllegalArgumentException e) { + throw new DorisConnectorException("Cannot deserialize Delta column mapping", e); + } + params.setCurrentSchemaId(carrier.getCurrentSchemaId()); + params.setHistorySchemaInfo(carrier.getHistorySchemaInfo()); + } + + static TSchema mappingSchema(StructType schema, boolean physical) { + TSchema result = new TSchema(); + result.setSchemaId(physical ? FILE_SCHEMA_ID : TABLE_SCHEMA_ID); + result.setRootField(mappingStruct(schema, physical)); + return result; + } + + private static TStructField mappingStruct(StructType schema, boolean physical) { + TStructField result = new TStructField(); + result.setFields(new ArrayList<>()); + for (StructField field : schema.fields()) { + FieldMetadata metadata = field.getMetadata(); + String physicalName = metadata.getString(PHYSICAL_NAME); + TField mapped = mappingField(field, physical ? physicalName : field.getName().toLowerCase(Locale.ROOT), + metadata.contains(NESTED_IDS) ? metadata.getMetadata(NESTED_IDS) : FieldMetadata.empty(), + physicalName, physical); + mapped.setId(Math.toIntExact(metadata.getLong(COLUMN_ID))); + result.addToFields(pointer(mapped)); + } + return result; + } + + private static TField mappingField(StructField field, String name, FieldMetadata nestedIds, + String path, boolean physical) { + TField result = new TField(); + result.setName(name); + result.setIsOptional(field.isNullable()); + DataType type = field.getDataType(); + TColumnType columnType = new TColumnType(); + columnType.setType(TPrimitiveType.STRING); + if (type instanceof StructType) { + columnType.setType(TPrimitiveType.STRUCT); + TNestedField nested = new TNestedField(); + nested.setStructField(mappingStruct((StructType) type, physical)); + result.setNestedField(nested); + } else if (type instanceof ArrayType) { + columnType.setType(TPrimitiveType.ARRAY); + TArrayField array = new TArrayField(); + array.setItemField(mappingCollectionField(((ArrayType) type).getElementField(), nestedIds, path, physical)); + TNestedField nested = new TNestedField(); + nested.setArrayField(array); + result.setNestedField(nested); + } else if (type instanceof MapType) { + columnType.setType(TPrimitiveType.MAP); + MapType mapType = (MapType) type; + TMapField map = new TMapField(); + map.setKeyField(mappingCollectionField(mapType.getKeyField(), nestedIds, path, physical)); + map.setValueField(mappingCollectionField(mapType.getValueField(), nestedIds, path, physical)); + TNestedField nested = new TNestedField(); + nested.setMapField(map); + result.setNestedField(nested); + } else if (type instanceof TimestampType || type instanceof TimestampNTZType) { + result.setTimestampIsAdjustedToUtc(type instanceof TimestampType); + } + result.setType(columnType); + return result; + } + + private static TFieldPtr mappingCollectionField(StructField field, FieldMetadata nestedIds, + String parentPath, boolean physical) { + String path = parentPath + "." + field.getName(); + TField result = mappingField(field, field.getName(), nestedIds, path, physical); + // Array/map element IDs are optional even in id mode. Iceberg-compatible writers store + // them under the nearest real struct field; retain them whenever they exist. + if (nestedIds.contains(path)) { + result.setId(Math.toIntExact(nestedIds.getLong(path))); + } + return pointer(result); + } + + private static TFieldPtr pointer(TField field) { + TFieldPtr pointer = new TFieldPtr(); + pointer.setFieldPtr(field); + return pointer; + } + +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCache.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCache.java new file mode 100644 index 00000000000000..74a3f7cb695745 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCache.java @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.engine.Engine; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +/** Catalog-owned immutable snapshots, isolated by user and fenced by the shared refresh registry. */ +final class DeltaLakeSnapshotCache { + static final String LATEST = "latest"; + static final String SNAPSHOT = "snapshot"; + private static final long DEFAULT_SNAPSHOT_BYTES = 256L << 20; + private final MetaCache latest; + private final MetaCache snapshots; + private final long latestTtlNanos; + private final LongSupplier nanoTime; + + DeltaLakeSnapshotCache(CatalogMetaCache owner, Map properties) { + this(owner, properties, System::nanoTime); + } + + DeltaLakeSnapshotCache(CatalogMetaCache owner, Map properties, LongSupplier nanoTime) { + this.nanoTime = nanoTime; + CacheSpec latestSpec = spec(properties, LATEST); + latestTtlNanos = latestSpec.getTtlSecond() > 0 + ? TimeUnit.SECONDS.toNanos(latestSpec.getTtlSecond()) : Long.MAX_VALUE; + // The framework's normal TTL is access-based. Apply fixed expiry below using the load-completion + // timestamp, while retaining its single-flight loads, invalidation fences and capacity governance. + long physicalTtl = latestSpec.getTtlSecond() == 0 ? 0 : CacheSpec.CACHE_NO_TTL; + CacheSpec physicalSpec = latestSpec.getMaxWeight().isPresent() + ? CacheSpec.ofWeight(latestSpec.isEnable(), physicalTtl, latestSpec.getCapacity(), + latestSpec.getMaxWeight().getAsLong()) + : CacheSpec.of(latestSpec.isEnable(), physicalTtl, latestSpec.getCapacity()); + latest = owner.create(MetaCacheDefinition.builder("deltalake-latest", physicalSpec, + key -> ScopePath.table(key.database, key.table)) + .sizeEstimator(MetaCacheSizeEstimators.reflective()).build()); + snapshots = owner.create(MetaCacheDefinition.builder("deltalake-snapshot", + spec(properties, SNAPSHOT), key -> ScopePath.table(key.table.database, key.table.table)) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(JvmSizeUtils.saturatedAdd( + value.bytes, key.estimatedBytes()))) + .build()); + } + + static CacheSpec spec(Map properties, String entry) { + Map configured = new HashMap<>(properties); + if (SNAPSHOT.equals(entry)) { + configured.putIfAbsent("meta.cache.deltalake.snapshot.max-weight", String.valueOf(DEFAULT_SNAPSHOT_BYTES)); + } + return CacheSpec.fromProperties(configured, "deltalake", entry, + CacheSpec.of(true, LATEST.equals(entry) ? 30 : 1800, 1000)); + } + + static void validate(Map properties) { + for (String entry : new String[] {LATEST, SNAPSHOT, DeltaLakeStatisticsCache.STATISTICS}) { + String prefix = "meta.cache.deltalake." + entry; + CacheSpec.checkBooleanProperty(properties.get(prefix + ".enable"), prefix + ".enable"); + CacheSpec.checkLongProperty(properties.get(prefix + ".ttl-second"), -1, prefix + ".ttl-second"); + CacheSpec.checkLongProperty(properties.get(prefix + ".capacity"), 0, prefix + ".capacity"); + } + CacheSpec.checkWeightProperties(properties, "deltalake", LATEST, SNAPSHOT, DeltaLakeStatisticsCache.STATISTICS); + } + + Snapshot latest(ConnectorSession session, String database, String table, String path, Engine engine) { + if (session == null) { + // Metadata callers without a user identity must not reuse another caller's cached result. + return loadLatest(path, engine); + } + return latest(new TableKey(session.getUser(), database, table, path), + () -> loadLatest(path, engine), + version -> TableManager.loadSnapshot(path).atVersion(version).build(engine)); + } + + Snapshot latest(TableKey key, Supplier loader, VersionLoader versionLoader) { + while (true) { + Snapshot[] loaded = {null}; + Latest reference = latest.get(key, ignored -> { + Snapshot snapshot = loader.get(); + DeltaLakeFeatureCheck.check(snapshot); + loaded[0] = snapshot; + SnapshotKey version = new SnapshotKey(key, DeltaKernelCompat.tableId(snapshot), snapshot.getVersion()); + snapshots.get(version, unused -> cached(snapshot)); + return new Latest(version.tableId, version.version, nanoTime.getAsLong()); + }); + if (latestTtlNanos != Long.MAX_VALUE && nanoTime.getAsLong() - reference.loadedAt >= latestTtlNanos) { + latest.compareAndSet(key, reference, null); + continue; + } + if (loaded[0] != null) { + // A disabled or weight-rejected snapshot cache must not force the miss caller to load twice. + return loaded[0]; + } + SnapshotKey version = new SnapshotKey(key, reference.tableId, reference.version); + return snapshots.get(version, ignored -> { + Snapshot snapshot = versionLoader.load(reference.version); + if (!reference.tableId.equals(DeltaKernelCompat.tableId(snapshot))) { + throw new DorisConnectorException("Delta table was recreated while its version was cached: " + + key.path); + } + DeltaLakeFeatureCheck.check(snapshot); + return cached(snapshot); + }).snapshot; + } + } + + private static Snapshot loadLatest(String path, Engine engine) { + Snapshot snapshot = TableManager.loadSnapshot(path).build(engine); + DeltaLakeFeatureCheck.check(snapshot); + return snapshot; + } + + private static CachedSnapshot cached(Snapshot snapshot) { + return new CachedSnapshot(snapshot, DeltaKernelCompat.snapshotRetainedBytes(snapshot)); + } + + interface VersionLoader { + Snapshot load(long version); + } + + static final class TableKey { + private final String user; + private final String database; + private final String table; + private final String path; + + TableKey(String user, String database, String table, String path) { + this.user = Objects.requireNonNull(user); + this.database = database; + this.table = table; + this.path = path; + } + + private long estimatedBytes() { + return 128L + JvmSizeUtils.stringSize(user) + JvmSizeUtils.stringSize(database) + + JvmSizeUtils.stringSize(table) + JvmSizeUtils.stringSize(path); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof TableKey)) { + return false; + } + TableKey other = (TableKey) object; + return user.equals(other.user) && database.equals(other.database) + && table.equals(other.table) && path.equals(other.path); + } + + @Override + public int hashCode() { + return Objects.hash(user, database, table, path); + } + } + + static final class SnapshotKey { + private final TableKey table; + private final String tableId; + private final long version; + + SnapshotKey(TableKey table, String tableId, long version) { + this.table = table; + this.tableId = tableId; + this.version = version; + } + + static SnapshotKey of(String user, DeltaLakeTableHandle handle) { + return new SnapshotKey(new TableKey(user, handle.getDatabase(), handle.getTable(), handle.getPath()), + handle.getTableId(), handle.getVersion()); + } + + ScopePath scope() { + return ScopePath.table(table.database, table.table); + } + + @Override + public String toString() { + return table.database + "." + table.table + " (" + tableId + ", version " + version + ")"; + } + + long estimatedBytes() { + return table.estimatedBytes() + JvmSizeUtils.stringSize(tableId) + 64; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof SnapshotKey)) { + return false; + } + SnapshotKey other = (SnapshotKey) object; + return table.equals(other.table) && tableId.equals(other.tableId) && version == other.version; + } + + @Override + public int hashCode() { + return Objects.hash(table, tableId, version); + } + } + + private static final class Latest { + private final String tableId; + private final long version; + private final long loadedAt; + + private Latest(String tableId, long version, long loadedAt) { + this.tableId = tableId; + this.version = version; + this.loadedAt = loadedAt; + } + } + + private static final class CachedSnapshot { + private final Snapshot snapshot; + private final long bytes; + + private CachedSnapshot(Snapshot snapshot, long bytes) { + this.snapshot = snapshot; + this.bytes = bytes; + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatementScope.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatementScope.java new file mode 100644 index 00000000000000..2ff61a7a5a2073 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatementScope.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScopes; + +import java.util.Optional; +import java.util.function.Supplier; + +final class DeltaLakeStatementScope { + static final String SNAPSHOT_NAMESPACE = "deltalake.snapshot"; + static final String VERSION_NAMESPACE = "deltalake.snapshot-version"; + + static final String FILE_COUNT_NAMESPACE = "deltalake.file-count"; + + /** Observed only at this statement's fixed table identity/version; never a cross-query cache. */ + static final class FileCount { + volatile long value = -1; + } + + static FileCount fileCount(ConnectorSession session, DeltaLakeTableHandle handle) { + return ConnectorStatementScopes.resolveInStatement(session, FILE_COUNT_NAMESPACE, handle.getDatabase(), + handle.getTable() + ":" + handle.getTableId() + ":" + handle.getVersion(), FileCount::new); + } + + private DeltaLakeStatementScope() { + } + + static Optional resolve(ConnectorSession session, String database, String table, + Supplier> loader) { + return ConnectorStatementScopes.resolveInStatement(session, SNAPSHOT_NAMESPACE, database, table, loader); + } + + static DeltaLakeTableHandle atVersion(ConnectorSession session, DeltaLakeTableHandle handle, long version, + Supplier loader) { + String key = String.join("\0", handle.getTable(), handle.getPath(), handle.getTableId(), + Long.toString(version)); + return ConnectorStatementScopes.resolveInStatement(session, VERSION_NAMESPACE, handle.getDatabase(), + key, loader); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCache.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCache.java new file mode 100644 index 00000000000000..5eb8321bd25bfa --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCache.java @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.deltalake.DeltaLakeSnapshotCache.SnapshotKey; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.engine.Engine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.FutureTask; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongConsumer; + +/** + * Nonblocking, user/table-identity/version-scoped statistics and observed file counts. + * Cache ownership also owns the pending work: + * REFRESH, eviction and catalog close cancel the old task, which can never publish into a replacement entry. + */ +final class DeltaLakeStatisticsCache { + static final String STATISTICS = "statistics"; + private static final Logger LOG = LogManager.getLogger(DeltaLakeStatisticsCache.class); + // Shared by this plugin's catalogs; never let a cold collection of large tables exhaust FE workers. + private static final ThreadPoolExecutor EXECUTOR = executor(); + private final MetaCache cache; + private final Executor executor; + + DeltaLakeStatisticsCache(CatalogMetaCache owner, Map properties) { + this(owner, properties, EXECUTOR); + } + + DeltaLakeStatisticsCache(CatalogMetaCache owner, Map properties, Executor executor) { + this.executor = executor; + cache = owner.create(MetaCacheDefinition.builder("deltalake-statistics", + DeltaLakeSnapshotCache.spec(properties, STATISTICS), SnapshotKey::scope) + // The callable retains only immutable table coordinates and catalog-owned services. + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512 + 2 * key.estimatedBytes())) + .removalListener((key, value, reason) -> cancel(value)) + .discardListener((key, value) -> cancel(value)) + .build()); + } + + Optional get(ConnectorSession session, DeltaLakeTableHandle handle, + DeltaLakeRuntime runtime) { + Optional fallback = + DeltaKernelCompat.tableStatistics(handle.snapshot(runtime.engine)); + if (fallback.isPresent() && fallback.get().getRowCount() == 0) { + return fallback; + } + if (session == null || !cache.isEnabled()) { + // Without an identity there is no authorization-safe cross-query cache key. + return fallback; + } + return get(SnapshotKey.of(session.getUser(), handle), fallback, loader(handle, runtime)); + } + + Optional get(SnapshotKey key, Optional fallback, + Callable loader) { + if (!cache.isEnabled()) { + return fallback; + } + Load load = cache.get(key, ignored -> new Load(key, loader)); + if (!load.task.isCancelled() && load.started.compareAndSet(false, true)) { + try { + // Submission is outside the cache's publication guards. A full queue returns the fallback; + // CallerRunsPolicy would turn a supposedly asynchronous million-file scan into planning IO. + executor.execute(load.task); + } catch (RejectedExecutionException e) { + cache.compareAndSet(key, load, null); + LOG.debug("Delta statistics queue is full; using size estimate"); + return fallback; + } + } + return completed(key, load).map(Statistics::tableStatistics).or(() -> fallback); + } + + OptionalLong cachedFileCount(ConnectorSession session, DeltaLakeTableHandle handle) { + // A planning hint must not create work, wait for IO, or share an authorization-sensitive result + // without a user identity. Reuse the exact same versioned entry as table statistics. + return session == null ? OptionalLong.empty() : cachedFileCount(SnapshotKey.of(session.getUser(), handle)); + } + + OptionalLong cachedFileCount(SnapshotKey key) { + Load load = cache.getIfPresent(key); + if (load == null) { + return OptionalLong.empty(); + } + long observed = load.observedFileCount; + return observed >= 0 ? OptionalLong.of(observed) + : completed(key, load).map(value -> OptionalLong.of(value.fileCount())).orElseGet(OptionalLong::empty); + } + + LongConsumer fileCountObserver(ConnectorSession session, DeltaLakeTableHandle handle, DeltaLakeRuntime runtime) { + return session == null ? count -> { } + : fileCountObserver(SnapshotKey.of(session.getUser(), handle), loader(handle, runtime)); + } + + LongConsumer fileCountObserver(SnapshotKey key, Callable loader) { + if (!cache.isEnabled()) { + return count -> { }; + } + // Capture the entry before scanning. A late report can update only this entry, never replace + // a newer entry after REFRESH/eviction. Creating it does not submit its statistics task. + Load load = cache.get(key, ignored -> new Load(key, loader)); + return count -> load.observedFileCount = count; + } + + private static Callable loader(DeltaLakeTableHandle handle, DeltaLakeRuntime runtime) { + DeltaLakeTableHandle coordinates = handle.withoutSnapshot(); + return () -> runtime.execute(() -> collect(coordinates.snapshot(runtime.engine), runtime.engine)); + } + + private Optional completed(SnapshotKey key, Load load) { + if (!load.task.isDone()) { + return Optional.empty(); + } + try { + return Optional.of(load.task.get()); + } catch (CancellationException e) { + return Optional.empty(); + } catch (ExecutionException e) { + // Failed loads are retryable, and cannot remove a newer entry installed after REFRESH. + cache.compareAndSet(key, load, null); + return Optional.empty(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DorisConnectorException("Interrupted retrieving Delta statistics", e); + } + } + + static Statistics collect(Snapshot snapshot, Engine engine) throws IOException { + DeltaLakeFeatureCheck.check(snapshot); + DeltaLakeRowCount rows = new DeltaLakeRowCount(); + long[] bytes = {0}; + long[] files = {0}; + DeltaLakeScanFiles.forEach(snapshot, engine, true, file -> { + rows.add(file); + bytes[0] = Math.addExact(bytes[0], DeltaKernelCompat.fileStatus(file).getSize()); + files[0]++; + }); + return new Statistics(new ConnectorTableStatistics(rows.value().orElse(-1), bytes[0]), files[0]); + } + + record Statistics(ConnectorTableStatistics tableStatistics, long fileCount) { + } + + private void cancel(Load load) { + load.task.cancel(true); + if (executor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) executor).remove(load.task); + } + } + + private static ThreadPoolExecutor executor() { + ThreadPoolExecutor pool = new ThreadPoolExecutor(2, 2, 60, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(64), action -> { + Thread thread = new Thread(action, "deltalake-statistics"); + thread.setDaemon(true); + thread.setContextClassLoader(DeltaLakeStatisticsCache.class.getClassLoader()); + return thread; + }); + pool.allowCoreThreadTimeOut(true); + return pool; + } + + private static final class Load { + private final AtomicBoolean started = new AtomicBoolean(); + private final FutureTask task; + private volatile long observedFileCount = -1; + + private Load(SnapshotKey key, Callable loader) { + task = new FutureTask<>(() -> { + try { + return loader.call(); + } catch (Exception | Error e) { + LOG.warn("Failed to load Delta table statistics for {}", key, e); + throw e; + } + }); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableHandle.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableHandle.java new file mode 100644 index 00000000000000..34a62c42f9f648 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableHandle.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.engine.Engine; + +/** Immutable statement coordinates; the in-memory snapshot avoids replay throughout planning. */ +final class DeltaLakeTableHandle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + + private final String database; + private final String table; + private final String path; + private final String tableId; + private final long version; + private final transient Snapshot snapshot; + + DeltaLakeTableHandle(String database, String table, Snapshot snapshot) { + this(database, table, snapshot.getPath(), DeltaKernelCompat.tableId(snapshot), snapshot.getVersion(), snapshot); + } + + private DeltaLakeTableHandle(String database, String table, String path, String tableId, + long version, Snapshot snapshot) { + this.database = database; + this.table = table; + this.path = path; + this.tableId = tableId; + this.version = version; + this.snapshot = snapshot; + } + + // Queued statistics work retains coordinates, not a potentially large cached snapshot. + DeltaLakeTableHandle withoutSnapshot() { + return new DeltaLakeTableHandle(database, table, path, tableId, version, null); + } + + String getDatabase() { + return database; + } + + String getTable() { + return table; + } + + String getPath() { + return path; + } + + String getTableId() { + return tableId; + } + + long getVersion() { + return version; + } + + DeltaLakeTableHandle atVersion(Engine engine, long requestedVersion) { + Snapshot pinned = TableManager.loadSnapshot(path).atVersion(requestedVersion).build(engine); + if (!tableId.equals(DeltaKernelCompat.tableId(pinned))) { + throw new DorisConnectorException("Delta table was recreated during the statement: " + path); + } + DeltaLakeFeatureCheck.check(pinned); + DeltaKernelCompat.prepareForConcurrentReads(pinned); + return new DeltaLakeTableHandle(database, table, pinned); + } + + Snapshot snapshot(Engine engine) { + if (snapshot != null) { + return snapshot; + } + Snapshot reloaded = TableManager.loadSnapshot(path).atVersion(version).build(engine); + // A recreated table can reuse version 0 at the same path; it is not the statement's table. + if (!tableId.equals(DeltaKernelCompat.tableId(reloaded))) { + throw new DorisConnectorException("Delta table was recreated during the statement: " + path); + } + return reloaded; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableResolver.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableResolver.java new file mode 100644 index 00000000000000..ff6eb5b668201a --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTableResolver.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +/** Resolves catalog names to table roots; schema and partitions are always read from the Delta log. */ +interface DeltaLakeTableResolver { + List listDatabases() throws IOException; + + List listTables(String database) throws IOException; + + Optional tableLocation(String database, String table) throws IOException; +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMapping.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMapping.java new file mode 100644 index 00000000000000..3652b44767d025 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMapping.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** Maps the current Delta schema; physical schema evolution remains the BE mapper's responsibility. */ +final class DeltaLakeTypeMapping { + private static final Map PRIMITIVES = Map.of( + BooleanType.BOOLEAN, "BOOLEAN", + ByteType.BYTE, "TINYINT", + ShortType.SHORT, "SMALLINT", + IntegerType.INTEGER, "INT", + LongType.LONG, "BIGINT", + FloatType.FLOAT, "FLOAT", + DoubleType.DOUBLE, "DOUBLE", + DateType.DATE, "DATEV2"); + + private DeltaLakeTypeMapping() { + } + + static ConnectorType toConnectorType(DataType type, boolean mapTimestampTz, boolean mapVarbinary) { + String primitive = PRIMITIVES.get(type); + if (primitive != null) { + return ConnectorType.of(primitive); + } + if (type instanceof StringType && ((StringType) type).isUTF8BinaryCollated()) { + return ConnectorType.of("STRING"); + } + if (type instanceof BinaryType) { + return ConnectorType.of(mapVarbinary ? "VARBINARY" : "STRING"); + } + if (type instanceof DecimalType) { + DecimalType decimal = (DecimalType) type; + return ConnectorType.of("DECIMALV3", decimal.getPrecision(), decimal.getScale()); + } + if (type instanceof TimestampType) { + return ConnectorType.of(mapTimestampTz ? "TIMESTAMPTZ" : "DATETIMEV2", 6, 0); + } + if (type instanceof TimestampNTZType) { + return ConnectorType.of("DATETIMEV2", 6, 0); + } + if (type instanceof ArrayType) { + ArrayType array = (ArrayType) type; + return ConnectorType.arrayOf(toConnectorType(array.getElementType(), mapTimestampTz, mapVarbinary), + array.containsNull()); + } + if (type instanceof MapType) { + MapType map = (MapType) type; + return ConnectorType.mapOf(toConnectorType(map.getKeyType(), mapTimestampTz, mapVarbinary), + toConnectorType(map.getValueType(), mapTimestampTz, mapVarbinary), + map.getValueField().isNullable()); + } + if (type instanceof StructType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List nullable = new ArrayList<>(); + List comments = new ArrayList<>(); + for (StructField field : ((StructType) type).fields()) { + names.add(field.getName().toLowerCase(Locale.ROOT)); + types.add(toConnectorType(field.getDataType(), mapTimestampTz, mapVarbinary)); + nullable.add(field.isNullable()); + comments.add(field.getMetadata().contains("comment") + ? field.getMetadata().getString("comment") : ""); + } + return ConnectorType.structOf(names, types, nullable, comments); + } + throw new DorisConnectorException("Unsupported Delta Lake type: " + type); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/TcclPinningConnectorContext.java b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/TcclPinningConnectorContext.java new file mode 100644 index 00000000000000..c5a97580736c6f --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/java/org/apache/doris/connector/deltalake/TcclPinningConnectorContext.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ForwardingConnectorContext; +import org.apache.doris.kerberos.HadoopAuthenticator; + +import java.util.concurrent.Callable; +import java.util.function.Supplier; + +/** Keep Kernel workers/reflection and Kerberos on this plugin's Hadoop class graph. */ +final class TcclPinningConnectorContext extends ForwardingConnectorContext { + private final Supplier authenticator; + + TcclPinningConnectorContext(ConnectorContext delegate, Supplier authenticator) { + super(delegate); + this.authenticator = authenticator; + } + + @Override + public T executeAuthenticated(Callable action) throws Exception { + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(TcclPinningConnectorContext.class.getClassLoader()); + HadoopAuthenticator auth = authenticator.get(); + // The parent engine's UGI is a different class in plugin mode. A Kerberos operation + // must authenticate through the plugin's own UGI rather than log in both copies. + return auth == null ? delegate().executeAuthenticated(action) : auth.doAs(action::call); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider b/fe/fe-connector/fe-connector-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider new file mode 100644 index 00000000000000..98887d2bc447af --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider @@ -0,0 +1 @@ +org.apache.doris.connector.deltalake.DeltaLakeConnectorProvider diff --git a/fe/fe-connector/fe-connector-deltalake/src/main/resources/deltalake.conf.template b/fe/fe-connector/fe-connector-deltalake/src/main/resources/deltalake.conf.template new file mode 100644 index 00000000000000..cb18979d6b6275 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/main/resources/deltalake.conf.template @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# Delta Lake currently has no process-wide settings. +# Set per-catalog options through CREATE CATALOG / ALTER CATALOG properties. diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelCompatTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelCompatTest.java new file mode 100644 index 00000000000000..6098a8133d34fd --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelCompatTest.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.deletionvectors.Base85Codec; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StructType; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +class DeltaKernelCompatTest { + @TempDir + Path warehouse; + + @Test + void resolvesAllDvStorageTypesOffsetsPaddingAndZeroCardinality() throws Exception { + createTable(); + UUID uuid = UUID.fromString("00000000-0000-0000-0000-000000000123"); + append("relative", "u", "sub/dir" + Base85Codec.encodeUUID(uuid), 7, 4, 3); + append("absolute", "p", "s3a://bucket/prefix/dv.bin", 9, 4, 3); + append("offsetless", "p", "s3a://bucket/prefix/dv.bin", null, 4, 3); + byte[] payload = {1, 2, 3}; + append("inline", "i", Base85Codec.encodeBytes(payload), null, 3, 3); + append("empty", "i", "", null, 0, 0); + try (DeltaLakeFileIO io = new DeltaLakeFileIO(new Configuration(), new RecordingConnectorContext())) { + Engine engine = DefaultEngine.create(io); + Snapshot snapshot = TableManager.loadSnapshot(warehouse.resolve("db/table").toString()).build(engine); + Map> descriptors = new HashMap<>(); + DeltaLakeScanFiles.forEach(snapshot, engine, true, row -> { + String path = DeltaKernelCompat.fileStatus(row).getPath(); + descriptors.put(path.substring(path.lastIndexOf('/') + 1), + DeltaKernelCompat.deletionVector(row, snapshot.getPath())); + Assertions.assertEquals(10, DeltaKernelCompat.numRecords(row).orElseThrow()); + }); + TDeltaLakeDeletionVectorDesc relative = descriptors.get("relative").orElseThrow(); + Assertions.assertTrue(relative.getPath().endsWith("/db/table/sub/dir/deletion_vector_" + uuid + ".bin")); + Assertions.assertEquals(7, relative.getOffset()); + Assertions.assertEquals(4, relative.getSizeInBytes()); + Assertions.assertEquals(3, relative.getCardinality()); + Assertions.assertEquals("s3a://bucket/prefix/dv.bin", descriptors.get("absolute").orElseThrow().getPath()); + Assertions.assertEquals(9, descriptors.get("absolute").orElseThrow().getOffset()); + Assertions.assertEquals(0, descriptors.get("offsetless").orElseThrow().getOffset()); + Assertions.assertArrayEquals(payload, descriptors.get("inline").orElseThrow().getInlineData()); + Assertions.assertFalse(descriptors.get("inline").orElseThrow().isSetPath()); + Assertions.assertTrue(descriptors.get("empty").isEmpty()); + } + TestingConnectorSession session = new TestingConnectorSession("dv-count"); + try (Connector connector = new DeltaLakeConnectorProvider().create(Map.of("warehouse", warehouse.toString()), + new RecordingConnectorContext()); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + Assertions.assertEquals(38, connector.getScanPlanProvider().planScan(session, + ConnectorScanRequest.builder(handle, Collections.emptyList()).countPushdown(true).build()) + .get(0).getPushDownRowCount()); + } + } + + @Test + void rejectsDeletionCardinalityLargerThanTheDataFileRowCount() throws Exception { + createTable(); + append("invalid", "p", "s3a://bucket/dv.bin", 0, 4, 11); + TestingConnectorSession session = new TestingConnectorSession("invalid-count"); + try (Connector connector = new DeltaLakeConnectorProvider().create(Map.of("warehouse", warehouse.toString()), + new RecordingConnectorContext()); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getScanPlanProvider().canServeMetadataOnlyCount(session, handle, Optional.empty())); + } + } + + private void createTable() throws Exception { + Path table = warehouse.resolve("db/table"); + DeltaLakeTestTables.commit(table, 0, "dv-table", new StructType().add("id", LongType.LONG)); + Path log = table.resolve("_delta_log/00000000000000000000.json"); + String contents = Files.readString(log); + String protocol = new ObjectMapper().writeValueAsString(Map.of("protocol", Map.of("minReaderVersion", 3, + "minWriterVersion", 7, "readerFeatures", new String[] {"deletionVectors"}, + "writerFeatures", new String[] {"deletionVectors"}))); + Files.writeString(log, protocol + contents.substring(contents.indexOf('\n'))); + } + + private void append(String name, String storage, String path, Integer offset, int size, long cardinality) + throws Exception { + Map dv = new HashMap<>(Map.of("storageType", storage, "pathOrInlineDv", path, + "sizeInBytes", size, "cardinality", cardinality)); + if (offset != null) { + dv.put("offset", offset); + } + String add = new ObjectMapper().writeValueAsString(Map.of("add", Map.of("path", name, "size", 10, + "partitionValues", Collections.emptyMap(), "modificationTime", 1L, "dataChange", true, + "stats", "{\"numRecords\":10}", "deletionVector", dv))); + Files.writeString(warehouse.resolve("db/table/_delta_log/00000000000000000000.json"), + add + "\n", StandardOpenOption.APPEND); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelSnapshotTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelSnapshotTest.java new file mode 100644 index 00000000000000..c58475c463ec9c --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaKernelSnapshotTest.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.data.Row; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.utils.CloseableIterator; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.stream.Stream; + +class DeltaKernelSnapshotTest { + @Test + void replaysClassicCheckpointWithDeletionVectors() throws Exception { + assertSnapshot("dv-partitioned-with-checkpoint", 15, 10, 15, 37, 2); + } + + @Test + void readsBothV2CheckpointRepresentationsIncludingSidecars() throws Exception { + assertSnapshot("v2-checkpoint-json", 2, 2, 4, 10, 0); + assertSnapshot("v2-checkpoint-parquet", 2, 2, 4, 10, 0); + } + + @Test + void confinesKernelInternalImportsToCompatibilityClass() throws Exception { + Path source = Path.of("src/main/java/org/apache/doris/connector/deltalake"); + try (Stream files = Files.walk(source)) { + for (Path file : (Iterable) files.filter(path -> path.toString().endsWith(".java"))::iterator) { + if (!file.getFileName().toString().equals("DeltaKernelCompat.java")) { + Assertions.assertFalse(Files.readString(file).contains("io.delta.kernel.internal"), + "Kernel internal dependency outside DeltaKernelCompat: " + file); + } + } + } + } + + private void assertSnapshot(String table, long version, long checkpoint, long files, + long records, long deleted) throws Exception { + try (DeltaLakeFileIO io = new DeltaLakeFileIO(new Configuration(), new RecordingConnectorContext())) { + assertSnapshot(DefaultEngine.create(io), table, version, checkpoint, files, records, deleted); + } + } + + private void assertSnapshot(Engine engine, String table, long version, long checkpoint, long files, + long records, long deleted) throws Exception { + Path path = Path.of(getClass().getResource("/deltalake/" + table).toURI()); + Snapshot snapshot = TableManager.loadSnapshot(path.toString()).build(engine); + DeltaLakeFeatureCheck.check(snapshot); + Assertions.assertEquals(version, snapshot.getVersion()); + Assertions.assertEquals(Optional.of(checkpoint), snapshot.getLogSegment().getCheckpointVersionOpt()); + long fileCount = 0; + long rowCount = 0; + long dvCount = 0; + try (CloseableIterator batches = + snapshot.getScanBuilder().build().getScanFiles(engine, true)) { + while (batches.hasNext()) { + try (CloseableIterator rows = batches.next().getRows()) { + while (rows.hasNext()) { + Row file = rows.next(); + Assertions.assertTrue(DeltaKernelCompat.fileStatus(file).getSize() > 0); + rowCount += DeltaKernelCompat.numRecords(file).orElseThrow(); + Optional dv = + DeltaKernelCompat.deletionVector(file, path.toString()); + dvCount += dv.map(TDeltaLakeDeletionVectorDesc::getCardinality).orElse(0L); + fileCount++; + } + } + } + } + Assertions.assertEquals(files, fileCount); + Assertions.assertEquals(records, rowCount); + Assertions.assertEquals(deleted, dvCount); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogPropertiesTest.java new file mode 100644 index 00000000000000..1513b1c6303beb --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeCatalogPropertiesTest.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.metastore.FileSystemMetaStoreProperties; +import org.apache.doris.connector.metastore.HmsMetaStoreProperties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +class DeltaLakeCatalogPropertiesTest { + @Test + void defaultsToFilesystemAndRequiresWarehouse() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> DeltaLakeCatalogProperties.of(Collections.emptyMap())); + DeltaLakeCatalogProperties properties = + DeltaLakeCatalogProperties.of(Collections.singletonMap("warehouse", "s3://bucket/warehouse")); + Assertions.assertEquals("filesystem", properties.getCatalogType()); + Assertions.assertInstanceOf(FileSystemMetaStoreProperties.class, properties.getMetastore()); + } + + @Test + void bindsHmsWithoutRequiringWarehouse() { + Map raw = new HashMap<>(); + raw.put("deltalake.catalog.type", "HMS"); + raw.put("hive.metastore.uris", "thrift://localhost:9083"); + DeltaLakeCatalogProperties properties = DeltaLakeCatalogProperties.of(raw); + Assertions.assertInstanceOf(HmsMetaStoreProperties.class, properties.getMetastore()); + Assertions.assertEquals("hms", properties.getCatalogType()); + raw.remove("hive.metastore.uris"); + Assertions.assertThrows(IllegalArgumentException.class, () -> DeltaLakeCatalogProperties.of(raw)); + } + + @Test + void rejectsUnknownMetastore() { + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> DeltaLakeCatalogProperties.of(Collections.singletonMap("deltalake.catalog.type", "glue"))); + Assertions.assertTrue(error.getMessage().contains("glue")); + } + + @Test + void bindsMappingOptionsAndCopiesProperties() { + Map raw = new HashMap<>(); + raw.put("warehouse", "/warehouse"); + raw.put("enable.mapping.timestamp_tz", "true"); + raw.put("enable.mapping.varbinary", "true"); + DeltaLakeCatalogProperties properties = DeltaLakeCatalogProperties.of(raw); + Assertions.assertTrue(properties.isMapTimestampTz()); + Assertions.assertTrue(properties.isMapVarbinary()); + raw.clear(); + Assertions.assertEquals("/warehouse", properties.getRaw().get("warehouse")); + } + + @Test + void validatesCachePropertiesAndRetainsSharedDisableSemantics() { + for (String entry : new String[] {"latest", "snapshot", "statistics"}) { + String prefix = "meta.cache.deltalake." + entry; + for (Map.Entry invalid : Map.of(".enable", "maybe", ".ttl-second", "-2", + ".capacity", "-1", ".max-weight", "0").entrySet()) { + String key = prefix + invalid.getKey(); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> DeltaLakeCatalogProperties.of(Map.of("warehouse", "/warehouse", + key, invalid.getValue()))); + Assertions.assertTrue(error.getMessage().contains(key), error.getMessage()); + } + Assertions.assertFalse(DeltaLakeSnapshotCache.spec(Map.of(prefix + ".enable", "false"), entry).isEnable()); + Assertions.assertEquals(0, DeltaLakeSnapshotCache.spec( + Map.of(prefix + ".ttl-second", "0"), entry).getTtlSecond()); + Assertions.assertEquals(-1, DeltaLakeSnapshotCache.spec( + Map.of(prefix + ".ttl-second", "-1"), entry).getTtlSecond()); + Assertions.assertEquals(1024, DeltaLakeSnapshotCache.spec( + Map.of(prefix + ".max-weight", "1KB"), entry).getMaxWeight().getAsLong()); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorTest.java new file mode 100644 index 00000000000000..9efe532a45bbad --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeConnectorTest.java @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorProvider; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.mvcc.ConnectorTimeTravelSpec; + +import io.delta.kernel.exceptions.KernelException; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.Set; + +class DeltaLakeConnectorTest { + private static final Map VARIANT_PROTOCOL = Map.of( + "minReaderVersion", 3, "minWriterVersion", 7, + "readerFeatures", List.of("variantType"), "writerFeatures", List.of("variantType")); + + @TempDir + Path warehouse; + + @Test + void providerDiscoversTablesAndReadsSchemaFromDeltaLog() throws Exception { + ConnectorProvider provider = ServiceLoader.load(ConnectorProvider.class).stream() + .map(ServiceLoader.Provider::get).filter(value -> "deltalake".equals(value.getType())) + .findFirst().orElseThrow(); + Assertions.assertNotNull(getClass().getResource("/" + provider.name() + ".conf.template")); + DeltaLakeTestTables.commit(warehouse.resolve("db/table"), 0, "first", + new StructType().add("ID", LongType.LONG), Collections.singletonList("ID")); + Files.createDirectories(warehouse.resolve("db/parquet_only")); + try (Connector connector = provider.create(Collections.singletonMap("warehouse", warehouse.toString()), + new RecordingConnectorContext()); + ConnectorMetadata metadata = connector.getMetadata(null)) { + Assertions.assertEquals(Collections.singletonList("db"), metadata.listDatabaseNames(null)); + Assertions.assertEquals(Collections.singletonList("table"), metadata.listTableNames(null, "db")); + Assertions.assertTrue(metadata.getTableHandle(null, "db", "parquet_only").isEmpty()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "table").orElseThrow(); + Assertions.assertTrue(connector.ownsHandle(handle)); + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + Assertions.assertEquals("ID", schema.getColumns().get(0).getName()); + Assertions.assertEquals("ID", schema.getProperties().get(ConnectorTableSchema.PARTITION_COLUMNS_KEY)); + Assertions.assertEquals(Collections.singleton("ID"), metadata.getColumnHandles(null, handle).keySet()); + Assertions.assertEquals("db", metadata.buildTableDescriptor(null, 1, "table", "db", + "table", 1, 42).getHiveTable().getDbName()); + } + } + + @Test + void statementPinsHandleSchemaAndMvccAcrossAppends() throws Exception { + Path table = warehouse.resolve("db/table"); + StructType original = new StructType().add("id", LongType.LONG); + DeltaLakeTestTables.commit(table, 0, "first", original); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(null)) { + TestingConnectorSession statement = new TestingConnectorSession("first"); + ConnectorTableHandle first = metadata.getTableHandle(statement, "db", "table").orElseThrow(); + ConnectorMvccSnapshot pin = metadata.beginQuerySnapshot(statement, first).orElseThrow(); + DeltaLakeTestTables.commit(table, 1, "first", original.add("added", StringType.STRING)); + Assertions.assertSame(first, metadata.getTableHandle(statement, "db", "table").orElseThrow()); + Assertions.assertEquals(1, metadata.getTableSchema(statement, first).getColumns().size()); + connector.invalidateTable("db", "table"); + TestingConnectorSession next = new TestingConnectorSession("next"); + ConnectorTableHandle latest = metadata.getTableHandle(next, "db", "table").orElseThrow(); + Assertions.assertEquals(2, metadata.getTableSchema(next, latest).getColumns().size()); + ConnectorTableHandle pinned = metadata.applySnapshot(next, latest, pin); + Assertions.assertEquals(0, ((DeltaLakeTableHandle) pinned).getVersion()); + Assertions.assertEquals(1, metadata.getTableSchema(next, latest, pin).getColumns().size()); + DeltaLakeTestTables.commit(warehouse.resolve("db/other"), 0, "other", original); + ConnectorTableHandle other = metadata.getTableHandle(next, "db", "other").orElseThrow(); + Assertions.assertThrows(DorisConnectorException.class, () -> metadata.applySnapshot(next, other, pin)); + } + } + + @Test + void readsExplicitVersionsWithTheirOwnSchemaAndColumnHandles() throws Exception { + Path table = warehouse.resolve("db/table"); + DeltaLakeTestTables.commit(table, 0, "first", new StructType().add("old_name", LongType.LONG)); + DeltaLakeTestTables.commit(table, 1, "first", new StructType().add("new_name", StringType.STRING)); + TestingConnectorSession session = new TestingConnectorSession("versioned-read"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle latest = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorMvccSnapshot old = metadata.resolveTimeTravel(session, latest, + ConnectorTimeTravelSpec.snapshotId("0")).orElseThrow(); + Assertions.assertEquals(0, old.getSnapshotId()); + Assertions.assertEquals(0, old.getSchemaId()); + Assertions.assertTrue(old.isSchemaRetained()); + Assertions.assertEquals("old_name", metadata.getTableSchema(session, latest, old) + .getColumns().get(0).getName()); + Assertions.assertTrue(metadata.supportsColumnHandleSnapshotPin(session)); + Assertions.assertEquals(Set.of("old_name"), metadata.getColumnHandles(session, latest, old).keySet()); + Assertions.assertSame(metadata.applySnapshot(session, latest, old), + metadata.applySnapshot(session, latest, old)); + Assertions.assertEquals(Set.of("new_name"), metadata.getColumnHandles(session, latest).keySet()); + + // An explicit version can already exist even when the statement's latest handle predates it. + DeltaLakeTestTables.commit(table, 2, "first", new StructType().add("later", LongType.LONG)); + ConnectorMvccSnapshot later = metadata.resolveTimeTravel(session, latest, + ConnectorTimeTravelSpec.snapshotId("2")).orElseThrow(); + Assertions.assertEquals(Set.of("later"), metadata.getColumnHandles(session, latest, later).keySet()); + Assertions.assertEquals(1, metadata.beginQuerySnapshot(session, latest).orElseThrow().getSnapshotId()); + } + } + + @Test + void rejectsInvalidOrUnavailableVersionsWithoutReadingLatest() throws Exception { + DeltaLakeTestTables.commit(warehouse.resolve("db/table"), 0, "id", new StructType().add("id", LongType.LONG)); + TestingConnectorSession session = new TestingConnectorSession("unavailable-version"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle table = metadata.getTableHandle(session, "db", "table").orElseThrow(); + for (ConnectorTimeTravelSpec spec : List.of(ConnectorTimeTravelSpec.versionRef("release"), + ConnectorTimeTravelSpec.versionRef("-1"), ConnectorTimeTravelSpec.snapshotId("-1"), + ConnectorTimeTravelSpec.snapshotId("9223372036854775808"))) { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.resolveTimeTravel(session, table, spec)); + Assertions.assertTrue(error.getMessage().contains("non-negative 64-bit integer: " + + spec.getStringValue()), error.getMessage()); + } + KernelException missing = Assertions.assertThrows(KernelException.class, () -> metadata.resolveTimeTravel( + session, table, ConnectorTimeTravelSpec.snapshotId("7"))); + Assertions.assertTrue(missing.getMessage().contains("Cannot load table version 7"), missing.getMessage()); + Assertions.assertTrue(metadata.resolveTimeTravel(session, table, + ConnectorTimeTravelSpec.timestamp("0", true)).isEmpty()); + } + } + + @Test + void rejectsVersionBeforeEarliestRetainedCheckpoint() throws Exception { + Path log = Files.createDirectories(warehouse.resolve("db/table/_delta_log")); + Path original = Path.of(getClass().getResource("/deltalake/dv-partitioned-with-checkpoint/_delta_log") + .toURI()); + for (String file : List.of("_last_checkpoint", "00000000000000000010.checkpoint.parquet", + "00000000000000000010.json")) { + Files.copy(original.resolve(file), log.resolve(file)); + } + TestingConnectorSession session = new TestingConnectorSession("truncated-version"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle latest = metadata.getTableHandle(session, "db", "table").orElseThrow(); + Assertions.assertEquals(10, metadata.beginQuerySnapshot(session, latest).orElseThrow().getSnapshotId()); + KernelException missing = Assertions.assertThrows(KernelException.class, () -> metadata.resolveTimeTravel( + session, latest, ConnectorTimeTravelSpec.snapshotId("9"))); + Assertions.assertTrue(missing.getMessage().contains("version 9"), missing.getMessage()); + } + } + + @Test + void checksReaderFeaturesAndIdentityOfRequestedVersion() throws Exception { + Path table = warehouse.resolve("db/table"); + StructType schema = new StructType().add("id", LongType.LONG); + DeltaLakeTestTables.commit(table, 0, "another-table", schema); + DeltaLakeTestTables.commit(table, 1, "first", schema, List.of(), VARIANT_PROTOCOL); + DeltaLakeTestTables.commit(table, 2, "first", schema); + TestingConnectorSession session = new TestingConnectorSession("unsupported-version"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle latest = metadata.getTableHandle(session, "db", "table").orElseThrow(); + DorisConnectorException identity = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.resolveTimeTravel(session, latest, ConnectorTimeTravelSpec.snapshotId("0"))); + Assertions.assertTrue(identity.getMessage().contains("recreated"), identity.getMessage()); + DorisConnectorException features = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.resolveTimeTravel(session, latest, ConnectorTimeTravelSpec.snapshotId("1"))); + Assertions.assertTrue(features.getMessage().contains("variantType"), features.getMessage()); + } + } + + @Test + void verifiesExplicitVersionEvenWhenItEqualsTheCachedLatestVersion() throws Exception { + Path table = warehouse.resolve("db/table"); + StructType schema = new StructType().add("id", LongType.LONG); + DeltaLakeTestTables.commit(table, 0, "first", schema); + TestingConnectorSession session = new TestingConnectorSession("recreated-at-version-zero"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle latest = metadata.getTableHandle(session, "db", "table").orElseThrow(); + DeltaLakeTestTables.commit(table, 0, "replacement", schema); + DorisConnectorException identity = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.resolveTimeTravel(session, latest, ConnectorTimeTravelSpec.snapshotId("0"))); + Assertions.assertTrue(identity.getMessage().contains("recreated"), identity.getMessage()); + } + } + + @Test + void rejectsUnsupportedLatestBeforeOpeningTheTable() throws Exception { + Path table = warehouse.resolve("db/table"); + StructType schema = new StructType().add("id", LongType.LONG); + DeltaLakeTestTables.commit(table, 0, "first", schema); + DeltaLakeTestTables.commit(table, 1, "first", schema, List.of(), VARIANT_PROTOCOL); + TestingConnectorSession session = new TestingConnectorSession("unsupported-latest"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + DorisConnectorException unsupported = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getTableHandle(session, "db", "table")); + Assertions.assertTrue(unsupported.getMessage().contains("Unsupported Delta Lake reader features"), + unsupported.getMessage()); + Assertions.assertTrue(unsupported.getMessage().contains("variantType"), unsupported.getMessage()); + } + } + + @Test + void closedUnusedConnectorCannotAllocateResources() throws Exception { + Connector connector = connector(); + connector.close(); + connector.close(); + Assertions.assertThrows(DorisConnectorException.class, () -> connector.getMetadata(null)); + } + + @Test + void statementNamespacesAreTypePrefixed() throws Exception { + String prefix = new DeltaLakeConnectorProvider().getType() + "."; + int checked = 0; + for (Field field : DeltaLakeStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) && field.getType() == String.class + && field.getName().endsWith("_NAMESPACE")) { + field.setAccessible(true); + Assertions.assertTrue(((String) field.get(null)).startsWith(prefix), field.getName()); + checked++; + } + } + Assertions.assertTrue(checked > 0); + } + + private Connector connector() { + return new DeltaLakeConnectorProvider().create(Collections.singletonMap("warehouse", warehouse.toString()), + new RecordingConnectorContext()); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheckTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheckTest.java new file mode 100644 index 00000000000000..80f74dc5462b83 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFeatureCheckTest.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.internal.actions.Protocol; +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TypeChange; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +class DeltaLakeFeatureCheckTest { + @Test + void legacyReaderOneDoesNotRequireWriterOnlyFeatures() { + Set readerFeatures = DeltaKernelCompat.readerFeatures(new Protocol(1, 7)); + Assertions.assertTrue(readerFeatures.isEmpty()); + DeltaLakeFeatureCheck.check("ordinary", readerFeatures); + } + + @Test + void legacyReaderTwoIncludesImplicitColumnMapping() { + Set features = DeltaKernelCompat.readerFeatures(new Protocol(2, 5)); + Assertions.assertEquals(Set.of("columnMapping"), features); + DeltaLakeFeatureCheck.check("mapping", features); + } + + @Test + void explicitReaderFeaturesDoNotIncludeWriterOnlyFeatures() { + Protocol protocol = new Protocol(3, 7, Set.of("deletionVectors"), + Set.of("deletionVectors", "appendOnly", "invariants")); + Assertions.assertEquals(Set.of("deletionVectors"), DeltaKernelCompat.readerFeatures(protocol)); + } + + @Test + void unsupportedFeatureErrorsIdentifyTableAndEveryFeature() { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> DeltaLakeFeatureCheck.check("warehouse/db/table", Set.of("variantType", "catalogManaged"))); + Assertions.assertTrue(error.getMessage().contains("warehouse/db/table")); + Assertions.assertTrue(error.getMessage().contains("variantType")); + Assertions.assertTrue(error.getMessage().contains("catalogManaged")); + DeltaLakeFeatureCheck.check("empty", Collections.emptySet()); + } + + @Test + void allowsOnlyVerifiedWideningFamilies() { + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(ByteType.BYTE, ShortType.SHORT)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(ShortType.SHORT, IntegerType.INTEGER)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(IntegerType.INTEGER, LongType.LONG)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(IntegerType.INTEGER, DoubleType.DOUBLE)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(ShortType.SHORT, new DecimalType(7, 2))); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(LongType.LONG, new DecimalType(21, 1))); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(FloatType.FLOAT, DoubleType.DOUBLE)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)); + Assertions.assertTrue(DeltaLakeFeatureCheck.supportedWidening(new DecimalType(10, 2), new DecimalType(12, 4))); + Assertions.assertFalse(DeltaLakeFeatureCheck.supportedWidening(LongType.LONG, DoubleType.DOUBLE)); + Assertions.assertFalse(DeltaLakeFeatureCheck.supportedWidening(LongType.LONG, new DecimalType(20, 2))); + Assertions.assertFalse(DeltaLakeFeatureCheck.supportedWidening(IntegerType.INTEGER, new DecimalType(10, 2))); + Assertions.assertFalse(DeltaLakeFeatureCheck.supportedWidening(new DecimalType(10, 2), new DecimalType(11, 4))); + Assertions.assertFalse(DeltaLakeFeatureCheck.supportedWidening(LongType.LONG, IntegerType.INTEGER)); + } + + @Test + void rejectsAnUnverifiedChangeInsideNestedCollectionsWithItsPath() { + StructField element = new StructField("element", DoubleType.DOUBLE, true) + .withTypeChanges(List.of(new TypeChange(LongType.LONG, DoubleType.DOUBLE))); + StructType schema = new StructType().add("items", new ArrayType(element)); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> DeltaLakeFeatureCheck.checkWidening(schema, "db.table")); + Assertions.assertTrue(error.getMessage().contains("db.table.items.element")); + Assertions.assertTrue(error.getMessage().contains("long -> double")); + } + +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileIOTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileIOTest.java new file mode 100644 index 00000000000000..4d979e376f3913 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileIOTest.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.FileStatus; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +class DeltaLakeFileIOTest { + @TempDir + Path directory; + + @Test + void listsSortedSuffixAndReadsSeekableInputs() throws Exception { + Files.writeString(directory.resolve("a.json"), "abcdef"); + Files.writeString(directory.resolve("c.json"), "c"); + Files.writeString(directory.resolve("b.json"), "b"); + Files.createDirectory(directory.resolve("b_directory")); + try (DeltaLakeFileIO io = fileIO(new RecordingConnectorContext())) { + List names = new ArrayList<>(); + try (CloseableIterator files = io.listFrom(directory.resolve("b.json").toString())) { + while (files.hasNext()) { + names.add(new org.apache.hadoop.fs.Path(files.next().getPath()).getName()); + } + } + Assertions.assertEquals(Arrays.asList("b.json", "c.json"), names); + String path = io.resolvePath(directory.resolve("a.json").toString()); + Assertions.assertEquals(6, io.getFileStatus(path).getSize()); + try (SeekableInputStream stream = io.newInputFile(path, 6).newStream()) { + stream.seek(2); + byte[] bytes = new byte[4]; + stream.readFully(bytes, 0, bytes.length); + Assertions.assertEquals("cdef", new String(bytes, StandardCharsets.UTF_8)); + } + Assertions.assertThrows(UnsupportedOperationException.class, () -> io.mkdirs(path)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> io.delete(path)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> io.newOutputFile(path)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> io.copyFileAtomically(path, path, true)); + } + } + + @Test + void sharesOnlyWithinCatalogAndClosesIdempotently() throws Exception { + RecordingConnectorContext context = new RecordingConnectorContext(); + DeltaLakeFileIO first = fileIO(context); + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(directory.toUri()); + try (DeltaLakeFileIO second = fileIO(new RecordingConnectorContext())) { + FileSystem expected = first.fileSystem(path); + ExecutorService executor = Executors.newFixedThreadPool(4); + try { + List> reads = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + reads.add(executor.submit(() -> first.fileSystem(path))); + } + for (Future read : reads) { + Assertions.assertSame(expected, read.get()); + } + } finally { + executor.shutdownNow(); + } + Assertions.assertEquals(1, context.authenticationCalls.get()); + Assertions.assertEquals(DeltaLakeFileIO.class.getClassLoader(), context.observedClassLoader); + FileSystem independent = second.fileSystem(path); + Assertions.assertNotSame(expected, independent); + first.close(); + first.close(); + Assertions.assertThrows(IOException.class, () -> first.fileSystem(path)); + Assertions.assertTrue(independent.exists(path)); + } finally { + first.close(); + } + } + + @Test + void restoresContextClassLoaderAfterFailure() { + RecordingConnectorContext delegate = new RecordingConnectorContext(); + TcclPinningConnectorContext context = new TcclPinningConnectorContext(delegate, () -> null); + ClassLoader original = Thread.currentThread().getContextClassLoader(); + IOException failure = new IOException("read failed"); + Assertions.assertSame(failure, Assertions.assertThrows(IOException.class, + () -> context.executeAuthenticated(() -> { + Assertions.assertEquals(TcclPinningConnectorContext.class.getClassLoader(), + Thread.currentThread().getContextClassLoader()); + throw failure; + }))); + Assertions.assertSame(original, Thread.currentThread().getContextClassLoader()); + } + + private DeltaLakeFileIO fileIO(RecordingConnectorContext context) { + Configuration conf = new Configuration(); + conf.setBoolean("fs.file.impl.disable.cache", true); + return new DeltaLakeFileIO(conf, new TcclPinningConnectorContext(context, () -> null)); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolverTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolverTest.java new file mode 100644 index 00000000000000..5c5de63fa1fcad --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeFileSystemTableResolverTest.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; + +class DeltaLakeFileSystemTableResolverTest { + @TempDir + Path warehouse; + + private final DeltaLakeFileIO fileIO = new DeltaLakeFileIO(new Configuration(), new RecordingConnectorContext()); + + @AfterEach + void close() throws Exception { + fileIO.close(); + } + + @Test + void listsOnlyImmediateDatabasesAndDeltaTables() throws Exception { + Files.createDirectories(warehouse.resolve("db/table/_delta_log")); + Files.createDirectories(warehouse.resolve("db/plain_parquet")); + Files.createDirectories(warehouse.resolve("other")); + Files.write(warehouse.resolve("README"), new byte[0]); + Files.write(warehouse.resolve("db/not_a_table"), new byte[0]); + DeltaLakeFileSystemTableResolver resolver = resolver(); + Assertions.assertEquals(Arrays.asList("db", "other"), resolver.listDatabases()); + Assertions.assertEquals(Collections.singletonList("table"), resolver.listTables("db")); + Assertions.assertTrue(resolver.tableLocation("db", "table").isPresent()); + Assertions.assertTrue(resolver.tableLocation("db", "plain_parquet").isEmpty()); + Assertions.assertTrue(resolver.tableLocation("db", "missing").isEmpty()); + } + + @Test + void emptyAndMissingNamespacesAreEmpty() throws Exception { + Assertions.assertTrue(resolver().listDatabases().isEmpty()); + Assertions.assertTrue(resolver().listTables("missing").isEmpty()); + Assertions.assertTrue(new DeltaLakeFileSystemTableResolver( + warehouse.resolve("not_created").toUri().toString(), fileIO).listDatabases().isEmpty()); + } + + @Test + void rejectsLogFilesAndNamesThatReplaceWarehouse() throws Exception { + Files.createDirectories(warehouse.resolve("db/broken")); + Files.write(warehouse.resolve("db/broken/_delta_log"), new byte[0]); + DeltaLakeFileSystemTableResolver resolver = resolver(); + Assertions.assertTrue(resolver.tableLocation("db", "broken").isEmpty()); + for (String name : Arrays.asList("", ".", "..", "../db", "/tmp", "s3://bucket", "a\\b")) { + Assertions.assertThrows(IllegalArgumentException.class, () -> resolver.listTables(name)); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolver.tableLocation("db", name)); + } + } + + private DeltaLakeFileSystemTableResolver resolver() { + return new DeltaLakeFileSystemTableResolver(warehouse.toUri().toString(), fileIO); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolverTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolverTest.java new file mode 100644 index 00000000000000..5899350ed182d5 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeHmsTableResolverTest.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsClientException; +import org.apache.doris.connector.hms.HmsDatabaseInfo; +import org.apache.doris.connector.hms.HmsPartitionInfo; +import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.spi.DorisConnectorException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +class DeltaLakeHmsTableResolverTest { + @Test + void onlyDeltaTablesAreVisibleAndSchemaIsNotReadFromHms() { + RecordingClient client = new RecordingClient(); + client.tables.put("delta", table("delta", "DeLtA", "s3://bucket/table")); + client.tables.put("hive", table("hive", "parquet", "s3://bucket/hive")); + DeltaLakeHmsTableResolver resolver = new DeltaLakeHmsTableResolver(client); + Assertions.assertEquals(Collections.singletonList("db"), resolver.listDatabases()); + Assertions.assertEquals(Collections.singletonList("delta"), resolver.listTables("db")); + Assertions.assertEquals("s3://bucket/table", resolver.tableLocation("db", "delta").orElseThrow()); + Assertions.assertTrue(resolver.tableLocation("db", "hive").isEmpty()); + Assertions.assertTrue(resolver.tableLocation("db", "missing").isEmpty()); + } + + @Test + void sparkSerdePathOverridesTheHmsPlaceholderLocation() { + RecordingClient client = new RecordingClient(); + client.tables.put("spark", HmsTableInfo.builder().dbName("db").tableName("spark") + .location("s3a://bucket/table-__PLACEHOLDER__") + .sdParameters(Collections.singletonMap("path", "s3a://bucket/table")) + .parameters(Collections.singletonMap("spark.sql.sources.provider", "delta")).build()); + DeltaLakeHmsTableResolver resolver = new DeltaLakeHmsTableResolver(client); + Assertions.assertEquals("s3a://bucket/table", resolver.tableLocation("db", "spark").orElseThrow()); + } + + @Test + void missingLocationAndMetastoreFailuresRemainErrors() { + RecordingClient client = new RecordingClient(); + client.tables.put("broken", table("broken", "delta", null)); + DeltaLakeHmsTableResolver resolver = new DeltaLakeHmsTableResolver(client); + Assertions.assertThrows(DorisConnectorException.class, () -> resolver.tableLocation("db", "broken")); + client.failure = new HmsClientException("metastore denied the request"); + Assertions.assertThrows(HmsClientException.class, () -> resolver.listTables("db")); + Assertions.assertThrows(HmsClientException.class, () -> resolver.tableLocation("db", "broken")); + } + + private static HmsTableInfo table(String name, String provider, String location) { + return HmsTableInfo.builder().dbName("db").tableName(name).location(location) + .parameters(Collections.singletonMap("spark.sql.sources.provider", provider)).build(); + } + + private static final class RecordingClient implements HmsClient { + private final Map tables = new LinkedHashMap<>(); + private HmsClientException failure; + + @Override + public List listDatabases() { + return Collections.singletonList("db"); + } + + @Override + public List listTables(String database) { + checkFailure(); + return new ArrayList<>(tables.keySet()); + } + + @Override + public boolean tableExists(String database, String table) { + checkFailure(); + return tables.containsKey(table); + } + + @Override + public HmsTableInfo getTable(String database, String table) { + checkFailure(); + return tables.get(table); + } + + private void checkFailure() { + if (failure != null) { + throw failure; + } + } + + @Override + public HmsDatabaseInfo getDatabase(String database) { + throw new UnsupportedOperationException(); + } + + @Override + public Map getDefaultColumnValues(String database, String table) { + throw new UnsupportedOperationException(); + } + + @Override + public List listPartitionNames(String database, String table, int maxParts) { + throw new UnsupportedOperationException(); + } + + @Override + public List getPartitions(String database, String table, List partitions) { + throw new UnsupportedOperationException(); + } + + @Override + public HmsPartitionInfo getPartition(String database, String table, List values) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValuesTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValuesTest.java new file mode 100644 index 00000000000000..28e94e6e1f8135 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePartitionValuesTest.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FieldMetadata; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TimeZone; + +class DeltaLakePartitionValuesTest { + @Test + void normalizesNumericDateAndBooleanValues() { + Assertions.assertEquals("-128", DeltaLakePartitionValues.normalizeValue(ByteType.BYTE, "-128")); + Assertions.assertEquals("32767", DeltaLakePartitionValues.normalizeValue(ShortType.SHORT, "32767")); + Assertions.assertEquals("42", DeltaLakePartitionValues.normalizeValue(IntegerType.INTEGER, "0042")); + Assertions.assertEquals("9223372036854775807", DeltaLakePartitionValues.normalizeValue( + LongType.LONG, "9223372036854775807")); + Assertions.assertEquals("1.25", DeltaLakePartitionValues.normalizeValue(FloatType.FLOAT, "1.25")); + Assertions.assertEquals("1.25", DeltaLakePartitionValues.normalizeValue(DoubleType.DOUBLE, "1.25")); + Assertions.assertEquals("100.00", DeltaLakePartitionValues.normalizeValue(new DecimalType(10, 2), "1.0000E2")); + Assertions.assertEquals("true", DeltaLakePartitionValues.normalizeValue(BooleanType.BOOLEAN, "TRUE")); + Assertions.assertEquals("2024-02-29", DeltaLakePartitionValues.normalizeValue(DateType.DATE, "2024-02-29")); + } + + @Test + void preservesNullEmptyBinaryAndPathCharacters() { + for (String value : Arrays.asList(null, "", "__HIVE_DEFAULT_PARTITION__", "a/b=c%20+d 中文\n")) { + Assertions.assertEquals(value, DeltaLakePartitionValues.normalizeValue(StringType.STRING, value)); + Assertions.assertEquals(value, DeltaLakePartitionValues.normalizeValue(BinaryType.BINARY, value)); + } + Assertions.assertNull(DeltaLakePartitionValues.normalizeValue(IntegerType.INTEGER, null)); + } + + @Test + void instantAndWallClockRenderingIgnoreJvmDefaultTimezone() { + TimeZone previous = TimeZone.getDefault(); + try { + for (String zone : Arrays.asList("UTC", "Asia/Shanghai", "America/Los_Angeles")) { + TimeZone.setDefault(TimeZone.getTimeZone(zone)); + for (String input : Arrays.asList("2024-03-10 02:30:00.123456", "2024-03-10T02:30:00.123456Z", + "2024-03-10T10:30:00.123456+08:00")) { + Assertions.assertEquals("2024-03-10T02:30:00.123456Z", + DeltaLakePartitionValues.normalizeValue(TimestampType.TIMESTAMP, input), zone); + } + Assertions.assertEquals("2024-03-10T02:30:00.123456", DeltaLakePartitionValues.normalizeValue( + TimestampNTZType.TIMESTAMP_NTZ, "2024-03-10 02:30:00.123456"), zone); + } + } finally { + TimeZone.setDefault(previous); + } + } + + @Test + void mapsPhysicalKeysToOrderedLogicalPartitionColumns() { + StructType schema = new StructType().add(new StructField("Renamed", IntegerType.INTEGER, true, + FieldMetadata.builder().putLong("delta.columnMapping.id", 1) + .putString("delta.columnMapping.physicalName", "col-uuid").build())) + .add(new StructField("Other", StringType.STRING, true, + FieldMetadata.builder().putLong("delta.columnMapping.id", 2) + .putString("delta.columnMapping.physicalName", "other-uuid").build())); + Map raw = new LinkedHashMap<>(); + raw.put("other-uuid", null); + raw.put("col-uuid", "0042"); + Map normalized = DeltaLakePartitionValues.normalize(schema, + Arrays.asList("Renamed", "Other"), true, raw); + Assertions.assertEquals(Arrays.asList("Renamed", "Other"), Arrays.asList(normalized.keySet().toArray())); + Assertions.assertEquals("42", normalized.get("Renamed")); + Assertions.assertNull(normalized.get("Other")); + Assertions.assertThrows(DorisConnectorException.class, () -> DeltaLakePartitionValues.normalize(schema, + Collections.singletonList("Renamed"), true, Collections.emptyMap())); + } + + @Test + void rejectsInvalidTypedValuesInsteadOfCoercingThem() { + for (String value : Arrays.asList("128", "", "null")) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> DeltaLakePartitionValues.normalizeValue(ByteType.BYTE, value)); + } + StructType schema = new StructType().add("flag", BooleanType.BOOLEAN); + Assertions.assertThrows(DorisConnectorException.class, () -> DeltaLakePartitionValues.normalize(schema, + Collections.singletonList("flag"), false, Collections.singletonMap("flag", "garbage"))); + Assertions.assertThrows(java.time.DateTimeException.class, + () -> DeltaLakePartitionValues.normalizeValue(DateType.DATE, "2023-02-29")); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningMetricsTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningMetricsTest.java new file mode 100644 index 00000000000000..8573fa506268eb --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningMetricsTest.java @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.thrift.TDeltaLakeFileDesc; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.data.Row; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.engine.ExpressionHandler; +import io.delta.kernel.engine.FileSystemClient; +import io.delta.kernel.engine.JsonHandler; +import io.delta.kernel.engine.MetricsReporter; +import io.delta.kernel.engine.ParquetHandler; +import io.delta.kernel.expressions.Column; +import io.delta.kernel.expressions.ExpressionEvaluator; +import io.delta.kernel.expressions.Literal; +import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.expressions.PredicateEvaluator; +import io.delta.kernel.metrics.ScanReport; +import io.delta.kernel.utils.CloseableIterator; +import io.delta.kernel.utils.FileStatus; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +/** Batch-boundary timing probe, separate from the actual-source scale acceptance. */ +@EnabledIfSystemProperty(named = "deltalake.planning.table", matches = ".+") +class DeltaLakePlanningMetricsTest { + @Test + void measuresReadDecodeReplayPruningAndRangeConstruction() throws Exception { + TimedEngine engine = new TimedEngine(DefaultEngine.create(new Configuration())); + Snapshot snapshot = TableManager.loadSnapshot(System.getProperty("deltalake.planning.table")).build(engine); + for (boolean filtered : new boolean[] {false, true}) { + engine.readNanos.set(0); + engine.parseNanos.set(0); + engine.expressionNanos.set(0); + Optional predicate = filtered + ? Optional.of(new Predicate("<", new Column("id"), Literal.ofLong(0))) : Optional.empty(); + long kernelNanos = 0; + long conversionNanos = 0; + long selected = 0; + long bytes = 0; + try (CloseableIterator files = DeltaLakeScanFiles.open(snapshot, engine, false, predicate)) { + while (true) { + long started = System.nanoTime(); + boolean more = files.hasNext(); + Row row = more ? files.next() : null; + kernelNanos += System.nanoTime() - started; + if (!more) { + break; + } + started = System.nanoTime(); + FileStatus file = DeltaKernelCompat.fileStatus(row); + DeltaLakeScanRange range = new DeltaLakeScanRange(file.getPath(), 0, file.getSize(), file.getSize(), + file.getModificationTime(), 128L << 20, Map.of(), new TDeltaLakeFileDesc()); + bytes += range.getLength(); + selected++; + conversionNanos += System.nanoTime() - started; + } + } + Assertions.assertEquals(filtered ? 0 : 1_000_000, selected); + Assertions.assertEquals(filtered ? 0 : 527_000_000, bytes); + Assertions.assertTrue(engine.report.getIsFullyConsumed()); + Assertions.assertEquals(1_000_000, engine.report.getScanMetrics().getNumActiveAddFiles()); + long read = engine.readNanos.get(); + long parse = engine.parseNanos.get(); + long expression = engine.expressionNanos.get(); + System.out.printf("DELTA_PHASES filtered=%s read_decode_ms=%.3f stats_parse_ms=%.3f " + + "expression_ms=%.3f replay_iteration_ms=%.3f range_dto_ms=%.3f selected=%d%n", + filtered, read / 1e6, parse / 1e6, expression / 1e6, + (kernelNanos - read - parse - expression) / 1e6, conversionNanos / 1e6, selected); + } + } + + private static final class TimedEngine implements Engine { + private final Engine delegate; + private final AtomicLong readNanos = new AtomicLong(); + private final AtomicLong parseNanos = new AtomicLong(); + private final AtomicLong expressionNanos = new AtomicLong(); + private ScanReport report; + + private TimedEngine(Engine delegate) { + this.delegate = delegate; + } + + @Override + public ExpressionHandler getExpressionHandler() { + return (ExpressionHandler) Proxy.newProxyInstance(ExpressionHandler.class.getClassLoader(), + new Class[] {ExpressionHandler.class}, (proxy, method, args) -> { + Object result = invoke(delegate.getExpressionHandler(), method, args); + if (result instanceof PredicateEvaluator) { + return timed(PredicateEvaluator.class, (PredicateEvaluator) result, expressionNanos); + } + if (result instanceof ExpressionEvaluator) { + return timed(ExpressionEvaluator.class, (ExpressionEvaluator) result, expressionNanos); + } + return result; + }); + } + + @Override + public JsonHandler getJsonHandler() { + return (JsonHandler) Proxy.newProxyInstance(JsonHandler.class.getClassLoader(), + new Class[] {JsonHandler.class}, (proxy, method, args) -> + measure(delegate.getJsonHandler(), method, args, + "parseJson".equals(method.getName()) ? parseNanos : readNanos)); + } + + @Override + public ParquetHandler getParquetHandler() { + return timed(ParquetHandler.class, delegate.getParquetHandler(), readNanos); + } + + @Override + public FileSystemClient getFileSystemClient() { + return delegate.getFileSystemClient(); + } + + @Override + public List getMetricsReporters() { + return List.of(value -> { + if (value instanceof ScanReport) { + report = (ScanReport) value; + } + }); + } + + private static T timed(Class type, T delegate, AtomicLong elapsed) { + return type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, + (proxy, method, args) -> measure(delegate, method, args, elapsed))); + } + + private static Object measure(Object delegate, Method method, Object[] args, AtomicLong elapsed) + throws Throwable { + long started = System.nanoTime(); + Object result; + try { + result = invoke(delegate, method, args); + } finally { + elapsed.addAndGet(System.nanoTime() - started); + } + if (result instanceof CloseableIterator) { + CloseableIterator iterator = (CloseableIterator) result; + return new CloseableIterator() { + @Override + public boolean hasNext() { + long started = System.nanoTime(); + try { + return iterator.hasNext(); + } finally { + elapsed.addAndGet(System.nanoTime() - started); + } + } + + @Override + public Object next() { + long started = System.nanoTime(); + try { + return iterator.next(); + } finally { + elapsed.addAndGet(System.nanoTime() - started); + } + } + + @Override + public void close() throws IOException { + long started = System.nanoTime(); + try { + iterator.close(); + } finally { + elapsed.addAndGet(System.nanoTime() - started); + } + } + }; + } + return result; + } + + private static Object invoke(Object delegate, Method method, Object[] args) throws Throwable { + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningScaleTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningScaleTest.java new file mode 100644 index 00000000000000..a8ecdef6e2fc7e --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePlanningScaleTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.scan.ConnectorSplitSource; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.lang.management.ManagementFactory; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Opt-in scale acceptance; use the planning-only checkpoint fixture, never scan its absent data objects. */ +@EnabledIfSystemProperty(named = "deltalake.planning.table", matches = ".+") +class DeltaLakePlanningScaleTest { + @Test + void collectsOneMillionFileStatisticsWithoutBlockingTheColdLookup() throws Exception { + Path table = Path.of(System.getProperty("deltalake.planning.table")).toAbsolutePath(); + try (Connector connector = new DeltaLakeConnectorProvider().create( + Map.of("warehouse", table.getParent().getParent().toString()), new RecordingConnectorContext())) { + TestingConnectorSession session = new TestingConnectorSession("statistics-scale"); + try (ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, + table.getParent().getFileName().toString(), table.getFileName().toString()).orElseThrow(); + long started = System.nanoTime(); + Optional result = metadata.getTableStatistics(session, handle); + long coldReturned = System.nanoTime(); + long peakHeap = 0; + while (result.isEmpty() || result.get().getRowCount() < 0) { + Assertions.assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(60), + "million-file statistics did not complete"); + Thread.sleep(10); + peakHeap = Math.max(peakHeap, ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed()); + result = metadata.getTableStatistics(session, handle); + } + Assertions.assertEquals(1_000_000, result.get().getRowCount()); + Assertions.assertEquals(527_000_000, result.get().getDataSize()); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(handle); + Assertions.assertEquals(1_000_000, + provider.streamingSplitEstimate(session, handle, Optional.empty(), false), + "completed statistics must keep large tables streaming"); + System.out.printf("DELTA_STATS_SCALE cold_ms=%.3f total_ms=%.3f rows=%d " + + "peak_heap_bytes=%d max_heap_bytes=%d%n", (coldReturned - started) / 1e6, + (System.nanoTime() - started) / 1e6, result.get().getRowCount(), peakHeap, + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + } + } + } + + @Test + void streamsOneMillionRangesInABoundedHeap() throws Exception { + Path table = Path.of(System.getProperty("deltalake.planning.table")).toAbsolutePath(); + String database = table.getParent().getFileName().toString(); + String name = table.getFileName().toString(); + try (Connector connector = new DeltaLakeConnectorProvider().create( + Map.of("warehouse", table.getParent().getParent().toString()), new RecordingConnectorContext())) { + for (int run = 0; run < 3; run++) { + TestingConnectorSession session = new TestingConnectorSession("scale-" + run, + Map.of("num_files_in_batch_mode", "1", "enable_external_scan_task_reuse", "true")); + long started = System.nanoTime(); + try (ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, database, name).orElseThrow(); + long snapshotDone = System.nanoTime(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(handle); + long estimate = provider.streamingSplitEstimate(session, handle, Optional.empty(), false); + Assertions.assertEquals(run == 0 ? 1 : 1_000_000, estimate, + "later queries reuse the complete count and remain streaming"); + long gateDone = System.nanoTime(); + long files = 0; + long bytes = 0; + long firstSplit = 0; + long peakHeap = 0; + try (ConnectorSplitSource source = provider.streamSplits(session, + ConnectorScanRequest.builder(handle, List.of()).build())) { + while (source.hasNext()) { + ConnectorScanRange range = source.next(); + if (files == 0) { + firstSplit = System.nanoTime(); + } + files++; + bytes += range.getLength(); + if (files % 10000 == 0) { + peakHeap = Math.max(peakHeap, + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed()); + } + } + } + long finished = System.nanoTime(); + Assertions.assertEquals(1_000_000, files); + Assertions.assertEquals(527_000_000, bytes); + System.out.printf("DELTA_SCALE run=%d snapshot_ms=%.3f gate_ms=%.3f first_split_ms=%.3f " + + "enumeration_ms=%.3f files=%d peak_heap_bytes=%d max_heap_bytes=%d%n", + run, (snapshotDone - started) / 1e6, (gateDone - snapshotDone) / 1e6, + (firstSplit - gateDone) / 1e6, (finished - gateDone) / 1e6, files, peakHeap, + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()); + } + } + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverterTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverterTest.java new file mode 100644 index 00000000000000..076bd405de25a6 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakePredicateConverterTest.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorAnd; +import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.spi.pushdown.ConnectorComparison; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.pushdown.ConnectorIn; +import org.apache.doris.connector.spi.pushdown.ConnectorIsNull; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.pushdown.ConnectorNot; +import org.apache.doris.connector.spi.pushdown.ConnectorOr; + +import io.delta.kernel.expressions.Predicate; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; +import io.delta.kernel.types.TypeChange; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +class DeltaLakePredicateConverterTest { + private final ConnectorColumnRef column = new ConnectorColumnRef("mixed", ConnectorType.of("BIGINT")); + private final DeltaLakePredicateConverter converter = new DeltaLakePredicateConverter( + new StructType().add("MiXeD", LongType.LONG), ZoneId.of("Asia/Shanghai")); + + @Test + void preservesLogicalColumnNamesAndEveryComparisonIncludingReversedOperands() { + for (ConnectorComparison.Operator operator : ConnectorComparison.Operator.values()) { + Predicate result = converter.convert(new ConnectorComparison(operator, column, + ConnectorLiteral.ofLong(1))).orElseThrow(); + Assertions.assertTrue(result.toString().contains("MiXeD")); + Assertions.assertTrue(converter.convert(new ConnectorComparison(operator, + ConnectorLiteral.ofLong(1), column)).isPresent()); + } + Predicate nullSafe = converter.convert(new ConnectorNot(new ConnectorComparison( + ConnectorComparison.Operator.EQ_FOR_NULL, column, ConnectorLiteral.ofLong(1)))).orElseThrow(); + Assertions.assertEquals("IS NOT DISTINCT FROM", ((Predicate) nullSafe.getChildren().get(0)).getName()); + } + + @Test + void onlyDropsUnconvertibleConjunctsOutsideNotAndOr() { + ConnectorExpression known = new ConnectorComparison(ConnectorComparison.Operator.EQ, + column, ConnectorLiteral.ofLong(1)); + ConnectorExpression unknown = new ConnectorComparison(ConnectorComparison.Operator.EQ, + column, ConnectorLiteral.ofString("one")); + ConnectorAnd and = new ConnectorAnd(List.of(known, unknown)); + Assertions.assertTrue(converter.convert(and).isPresent()); + Assertions.assertTrue(converter.convert(new ConnectorNot(and)).isEmpty()); + Assertions.assertTrue(converter.convert(new ConnectorOr(List.of(known, unknown))).isEmpty()); + Assertions.assertTrue(converter.convert(new ConnectorOr(List.of(known, and))).isEmpty()); + Assertions.assertTrue(converter.convert(new ConnectorNot(new ConnectorOr(List.of(known, known)))).isPresent()); + } + + @Test + void preservesInNullAndNegationSemantics() { + for (boolean negated : List.of(false, true)) { + Assertions.assertTrue(converter.convert(new ConnectorIsNull(column, negated)).isPresent()); + Assertions.assertTrue(converter.convert(new ConnectorIn(column, + List.of(ConnectorLiteral.ofLong(1), ConnectorLiteral.ofNull(ConnectorType.of("BIGINT"))), + negated)).isPresent()); + Assertions.assertTrue(converter.convert(new ConnectorIn(column, + List.of(ConnectorLiteral.ofString("one")), negated)).isEmpty()); + } + } + + @Test + void acceptsExactPrimitiveLiteralsAndRejectsLossyOrUnsupportedConversions() { + Assertions.assertEquals(true, converter.literal(BooleanType.BOOLEAN, ConnectorLiteral.ofBoolean(true)).getValue()); + Assertions.assertEquals((byte) -128, converter.literal(ByteType.BYTE, ConnectorLiteral.ofInt(-128)).getValue()); + Assertions.assertEquals((short) 32767, converter.literal(ShortType.SHORT, ConnectorLiteral.ofInt(32767)).getValue()); + Assertions.assertEquals(7, converter.literal(IntegerType.INTEGER, ConnectorLiteral.ofLong(7)).getValue()); + Assertions.assertEquals(Long.MAX_VALUE, converter.literal(LongType.LONG, + ConnectorLiteral.ofLong(Long.MAX_VALUE)).getValue()); + Assertions.assertEquals(1.25f, converter.literal(FloatType.FLOAT, + new ConnectorLiteral(ConnectorType.of("FLOAT"), 1.25)).getValue()); + Assertions.assertEquals(1.25, converter.literal(DoubleType.DOUBLE, ConnectorLiteral.ofDouble(1.25)).getValue()); + Assertions.assertEquals("a/b%+中文", converter.literal(StringType.STRING, + ConnectorLiteral.ofString("a/b%+中文")).getValue()); + Assertions.assertArrayEquals(new byte[] {0, -1}, (byte[]) converter.literal(BinaryType.BINARY, + new ConnectorLiteral(ConnectorType.of("VARBINARY"), new byte[] {0, -1})).getValue()); + Assertions.assertEquals(-1, converter.literal(DateType.DATE, + ConnectorLiteral.ofDate(LocalDate.of(1969, 12, 31))).getValue()); + Assertions.assertEquals(new BigDecimal("1.20"), converter.literal(new DecimalType(10, 2), + ConnectorLiteral.ofDecimal(new BigDecimal("1.2"), 2, 1)).getValue()); + Assertions.assertNull(converter.literal(ByteType.BYTE, ConnectorLiteral.ofInt(128))); + Assertions.assertNull(converter.literal(IntegerType.INTEGER, ConnectorLiteral.ofLong(Long.MAX_VALUE))); + Assertions.assertNull(converter.literal(LongType.LONG, ConnectorLiteral.ofDouble(1.1))); + Assertions.assertNull(converter.literal(LongType.LONG, ConnectorLiteral.ofDouble(1.0e18))); + Assertions.assertNull(converter.literal(new DecimalType(20, 0), ConnectorLiteral.ofDouble(1.0e18))); + Assertions.assertNull(converter.literal(FloatType.FLOAT, ConnectorLiteral.ofDouble(0.1))); + Assertions.assertNull(converter.literal(DoubleType.DOUBLE, ConnectorLiteral.ofDouble(Double.NaN))); + Assertions.assertNull(converter.literal(new DecimalType(3, 2), + ConnectorLiteral.ofDecimal(new BigDecimal("12.345"), 5, 3))); + Assertions.assertNull(converter.literal(LongType.LONG, ConnectorLiteral.ofString("1"))); + } + + @Test + void distinguishesUtcTimestampCarriersFromWallClockAndNtz() { + LocalDateTime epoch = LocalDateTime.of(1970, 1, 1, 0, 0); + Assertions.assertEquals(-28_800_000_000L, converter.literal(TimestampType.TIMESTAMP, + ConnectorLiteral.ofDatetime(epoch)).getValue()); + Assertions.assertEquals(0L, converter.literal(TimestampType.TIMESTAMP, + new ConnectorLiteral(ConnectorType.of("TIMESTAMPTZ", 6, 0), epoch)).getValue()); + Assertions.assertEquals(0L, converter.literal(TimestampNTZType.TIMESTAMP_NTZ, + ConnectorLiteral.ofDatetime(epoch)).getValue()); + Assertions.assertEquals(-1L, converter.literal(TimestampNTZType.TIMESTAMP_NTZ, + ConnectorLiteral.ofDatetime(epoch.minusNanos(1000))).getValue()); + } + + @Test + void leavesDateToNtzPredicatesResidualBecauseOldStatsUseDateStrings() { + StructField field = new StructField("dt", TimestampNTZType.TIMESTAMP_NTZ, true) + .withTypeChanges(List.of(new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ))); + DeltaLakePredicateConverter widening = new DeltaLakePredicateConverter( + new StructType().add(field).add("MiXeD", LongType.LONG), ZoneId.of("UTC")); + ConnectorExpression date = new ConnectorComparison(ConnectorComparison.Operator.GT, + new ConnectorColumnRef("dt", ConnectorType.of("DATETIMEV2", 6, 0)), + ConnectorLiteral.ofDatetime(LocalDateTime.of(2024, 2, 29, 0, 0))); + ConnectorExpression integer = new ConnectorComparison(ConnectorComparison.Operator.EQ, + column, ConnectorLiteral.ofLong(1)); + Assertions.assertTrue(widening.convert(date).isEmpty()); + Assertions.assertEquals(widening.convert(integer).orElseThrow().toString(), + widening.convert(new ConnectorAnd(List.of(date, integer))).orElseThrow().toString()); + Assertions.assertTrue(widening.convert(new ConnectorOr(List.of(date, integer))).isEmpty()); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeRuntimeTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeRuntimeTest.java new file mode 100644 index 00000000000000..5f25dd3eb1b9d3 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeRuntimeTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.metastore.spi.AbstractHmsMetaStoreProperties; +import org.apache.doris.connector.spi.ConnectorStorageContext; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.filesystem.FileSystemType; +import org.apache.doris.filesystem.properties.HadoopStorageProperties; +import org.apache.doris.filesystem.properties.StorageKind; +import org.apache.doris.filesystem.properties.StorageProperties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +class DeltaLakeRuntimeTest { + @Test + void storageConfigurationUsesTypedPropertiesAndExplicitOverrides() { + RecordingConnectorContext context = new RecordingConnectorContext() { + @Override + public ConnectorStorageContext getStorageContext() { + return new ConnectorStorageContext() { + @Override + public List getStorageProperties() { + return Collections.singletonList(new TestStorageProperties()); + } + }; + } + }; + Configuration conf = DeltaLakeRuntime.storageConfiguration( + Map.of("fs.s3a.endpoint", "http://explicit:9000", "dfs.client.socket-timeout", "1234"), context); + Assertions.assertEquals("http://explicit:9000", conf.get("fs.s3a.endpoint")); + Assertions.assertEquals("true", conf.get("fs.s3a.path.style.access")); + Assertions.assertEquals("1234", conf.get("dfs.client.socket-timeout")); + Assertions.assertSame(DeltaLakeRuntime.class.getClassLoader(), conf.getClassLoader()); + } + + @Test + void hmsSimpleIdentityDoesNotUseStorageUsername() throws Exception { + DeltaLakeCatalogProperties properties = DeltaLakeCatalogProperties.of(Map.of( + "deltalake.catalog.type", "hms", "hive.metastore.uris", "thrift://localhost:9083", + "hive.metastore.authentication.type", "simple", "hive.metastore.username", "delta-metastore", + "hadoop.username", "delta-storage")); + Assertions.assertEquals("delta-metastore", DeltaLakeRuntime.hmsAuthenticator( + (AbstractHmsMetaStoreProperties) properties.getMetastore()).getUGI().getUserName()); + } + + @Test + void rejectsInvalidPoolSizeBeforeConnecting() { + Assertions.assertThrows(IllegalArgumentException.class, () -> DeltaLakeCatalogProperties.of( + Map.of("warehouse", "/warehouse", "hive.metastore.client.pool.size", "-1"))); + } + + @Test + void explicitStorageUserReachesPluginUgi() throws Exception { + Configuration configuration = new Configuration(); + configuration.set("hadoop.username", "delta-storage"); + RecordingConnectorContext delegate = new RecordingConnectorContext(); + TcclPinningConnectorContext context = new TcclPinningConnectorContext(delegate, + () -> DeltaLakeRuntime.storageAuthenticator(configuration)); + Assertions.assertEquals("delta-storage", context.executeAuthenticated( + () -> UserGroupInformation.getCurrentUser().getUserName())); + Assertions.assertEquals(0, delegate.authenticationCalls.get()); + } + + @Test + void metadataInterruptionRestoresInterruptFlag() throws Exception { + DeltaLakeCatalogProperties properties = DeltaLakeCatalogProperties.of(Map.of("warehouse", "/warehouse")); + try (DeltaLakeRuntime runtime = new DeltaLakeRuntime(properties, new RecordingConnectorContext())) { + Assertions.assertThrows(DorisConnectorException.class, () -> runtime.execute(() -> { + throw new InterruptedException("query cancelled"); + })); + Assertions.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + private static final class TestStorageProperties implements StorageProperties, HadoopStorageProperties { + @Override + public Optional toHadoopProperties() { + return Optional.of(this); + } + + @Override + public Map toHadoopConfigurationMap() { + return Map.of("fs.s3a.endpoint", "http://normalized:9000", "fs.s3a.path.style.access", "true"); + } + + @Override + public String providerName() { + return "TEST"; + } + + @Override + public StorageKind kind() { + return StorageKind.HDFS_COMPATIBLE; + } + + @Override + public FileSystemType type() { + return FileSystemType.HDFS; + } + + @Override + public Map rawProperties() { + return Collections.emptyMap(); + } + + @Override + public Map matchedProperties() { + return Collections.emptyMap(); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanFilesTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanFilesTest.java new file mode 100644 index 00000000000000..6c398e2082a13c --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanFilesTest.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.FilteredColumnarBatch; +import io.delta.kernel.data.Row; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +class DeltaLakeScanFilesTest { + @Test + void closesBothLevelsAtEofAndDoesNotOpenAnUnusedSource() throws Exception { + AtomicInteger opened = new AtomicInteger(); + DeltaLakeScanFiles.Files unused = new DeltaLakeScanFiles.Files(() -> { + opened.incrementAndGet(); + return Assertions.fail("unused source opened"); + }); + unused.close(); + Assertions.assertFalse(unused.hasNext()); + Assertions.assertEquals(0, opened.get()); + AtomicInteger rowsClosed = new AtomicInteger(); + AtomicInteger batchesClosed = new AtomicInteger(); + try (DeltaLakeScanFiles.Files files = source(rowsClosed, batchesClosed, new AtomicBoolean())) { + Assertions.assertTrue(files.hasNext()); + Assertions.assertNotNull(files.next()); + Assertions.assertFalse(files.hasNext()); + Assertions.assertEquals(1, rowsClosed.get()); + Assertions.assertEquals(1, batchesClosed.get()); + } + Assertions.assertEquals(1, rowsClosed.get()); + Assertions.assertEquals(1, batchesClosed.get()); + } + + @Test + void closesBothLevelsOnEarlyCloseAndIteratorFailure() throws Exception { + AtomicInteger rowsClosed = new AtomicInteger(); + AtomicInteger batchesClosed = new AtomicInteger(); + AtomicBoolean fail = new AtomicBoolean(); + try (DeltaLakeScanFiles.Files files = source(rowsClosed, batchesClosed, fail)) { + Assertions.assertTrue(files.hasNext()); + fail.set(true); + Assertions.assertThrows(IllegalStateException.class, files::hasNext); + Assertions.assertEquals(1, rowsClosed.get()); + Assertions.assertEquals(1, batchesClosed.get()); + } + fail.set(false); + try (DeltaLakeScanFiles.Files files = source(rowsClosed, batchesClosed, fail)) { + Assertions.assertTrue(files.hasNext()); + } + Assertions.assertEquals(2, rowsClosed.get()); + Assertions.assertEquals(2, batchesClosed.get()); + } + + @Test + void cancellationClosesIteratorsAndPreservesTheInterrupt() throws Exception { + AtomicInteger rowsClosed = new AtomicInteger(); + AtomicInteger batchesClosed = new AtomicInteger(); + try (DeltaLakeScanFiles.Files files = source(rowsClosed, batchesClosed, new AtomicBoolean())) { + Assertions.assertTrue(files.hasNext()); + Thread.currentThread().interrupt(); + Assertions.assertThrows(DorisConnectorException.class, files::hasNext); + Assertions.assertTrue(Thread.currentThread().isInterrupted()); + Assertions.assertEquals(1, rowsClosed.get()); + Assertions.assertEquals(1, batchesClosed.get()); + } finally { + Thread.interrupted(); + } + } + + private DeltaLakeScanFiles.Files source(AtomicInteger rowsClosed, AtomicInteger batchesClosed, AtomicBoolean fail) { + ColumnarBatch data = new ColumnarBatch() { + @Override + public StructType getSchema() { + return new StructType(); + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + throw new IndexOutOfBoundsException(); + } + + @Override + public int getSize() { + return 1; + } + }; + FilteredColumnarBatch batch = new FilteredColumnarBatch(data, Optional.empty()) { + @Override + public CloseableIterator getRows() { + return tracked(data.getRows(), rowsClosed, fail); + } + }; + return new DeltaLakeScanFiles.Files(() -> tracked(List.of(batch).iterator(), batchesClosed, + new AtomicBoolean())); + } + + private static CloseableIterator tracked(Iterator delegate, AtomicInteger closed, AtomicBoolean fail) { + return new CloseableIterator<>() { + @Override + public boolean hasNext() { + if (fail.get()) { + throw new IllegalStateException("injected iterator failure"); + } + return delegate.hasNext(); + } + + @Override + public T next() { + return delegate.next(); + } + + @Override + public void close() throws IOException { + closed.incrementAndGet(); + if (delegate instanceof CloseableIterator) { + ((CloseableIterator) delegate).close(); + } + } + }; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemoTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemoTest.java new file mode 100644 index 00000000000000..ba46fd4305add1 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanMemoTest.java @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.scan.ConnectorScanContext; +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; +import org.apache.doris.thrift.TDeltaLakeFileDesc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +class DeltaLakeScanPlanMemoTest { + private final DeltaLakeScanPlanMemo.Key key = key("table"); + private final DeltaLakeScanPlanMemo.Plan plan = new DeltaLakeScanPlanMemo.Plan(List.of(range("a", 0)), null); + + @Test + void coalescesConcurrentLoadsWithoutLettingACancelledWaiterCancelTheOwner() throws Exception { + DeltaLakeScanPlanMemo memo = new DeltaLakeScanPlanMemo(); + AtomicInteger loads = new AtomicInteger(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch waiting = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(3); + try { + Future owner = executor.submit(() -> memo.get(key, + new ConnectorScanContext(), () -> { + loads.incrementAndGet(); + started.countDown(); + await(release); + return plan; + })); + await(started); + ConnectorScanContext cancelled = new ConnectorScanContext(); + Future waiter = executor.submit(() -> { + waiting.countDown(); + return memo.get(key, cancelled, () -> { + Assertions.fail("waiter must not load"); + return plan; + }); + }); + await(waiting); + Assertions.assertThrows(TimeoutException.class, () -> waiter.get(100, TimeUnit.MILLISECONDS)); + cancelled.cancel(); + Assertions.assertInstanceOf(CancellationException.class, + Assertions.assertThrows(ExecutionException.class, () -> waiter.get(10, TimeUnit.SECONDS)).getCause()); + Future survivor = executor.submit(() -> memo.get(key, + new ConnectorScanContext(), () -> { + Assertions.fail("concurrent identical scan must share the load"); + return plan; + })); + // A different key must make progress while the first owner's IO is blocked. + Assertions.assertFalse(memo.get(key("another-table"), new ConnectorScanContext(), () -> plan).reused()); + release.countDown(); + Assertions.assertFalse(owner.get(10, TimeUnit.SECONDS).reused()); + Assertions.assertSame(plan, survivor.get(10, TimeUnit.SECONDS).plan()); + Assertions.assertTrue(survivor.get().reused()); + Assertions.assertEquals(1, loads.get()); + } finally { + release.countDown(); + executor.shutdownNow(); + Assertions.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test + void interruptedWaitersLeaveAndOwnerFailuresReachOtherWaitersWithoutBeingCached() throws Exception { + DeltaLakeScanPlanMemo memo = new DeltaLakeScanPlanMemo(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch waiting = new CountDownLatch(1); + AtomicReference waiterThread = new AtomicReference<>(); + AtomicBoolean interrupted = new AtomicBoolean(); + IllegalStateException failure = new IllegalStateException("checkpoint IO failed"); + ExecutorService executor = Executors.newFixedThreadPool(3); + try { + Future owner = executor.submit(() -> memo.get(key, new ConnectorScanContext(), () -> { + started.countDown(); + await(release); + throw failure; + })); + await(started); + Future waiter = executor.submit(() -> { + waiterThread.set(Thread.currentThread()); + waiting.countDown(); + try { + return memo.get(key, new ConnectorScanContext(), () -> plan); + } finally { + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + await(waiting); + Assertions.assertThrows(TimeoutException.class, () -> waiter.get(100, TimeUnit.MILLISECONDS)); + waiterThread.get().interrupt(); + Assertions.assertInstanceOf(CancellationException.class, + Assertions.assertThrows(ExecutionException.class, () -> waiter.get(10, TimeUnit.SECONDS)).getCause()); + Assertions.assertTrue(interrupted.get()); + Future other = executor.submit(() -> memo.get(key, new ConnectorScanContext(), () -> plan)); + Assertions.assertThrows(TimeoutException.class, () -> other.get(100, TimeUnit.MILLISECONDS)); + release.countDown(); + for (Future task : List.of(owner, other)) { + Assertions.assertSame(failure, + Assertions.assertThrows(ExecutionException.class, () -> task.get(10, TimeUnit.SECONDS)).getCause()); + } + Assertions.assertFalse(memo.get(key, new ConnectorScanContext(), () -> plan).reused()); + } finally { + release.countDown(); + executor.shutdownNow(); + Assertions.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test + void failedAndCancelledLoadsAreRetriedInsteadOfPublishingPartialPlans() { + DeltaLakeScanPlanMemo memo = new DeltaLakeScanPlanMemo(); + IllegalStateException failure = new IllegalStateException("failed after enumerating one file"); + Assertions.assertSame(failure, Assertions.assertThrows(IllegalStateException.class, + () -> memo.get(key, new ConnectorScanContext(), () -> { + throw failure; + }))); + ConnectorScanContext cancelled = new ConnectorScanContext(); + Assertions.assertThrows(CancellationException.class, () -> memo.get(key, cancelled, () -> { + cancelled.cancel(); + return plan; + })); + Assertions.assertFalse(memo.get(key, new ConnectorScanContext(), () -> plan).reused()); + Assertions.assertTrue(memo.get(key, new ConnectorScanContext(), () -> plan).reused()); + Thread.currentThread().interrupt(); + try { + Assertions.assertThrows(CancellationException.class, + () -> memo.get(key, new ConnectorScanContext(), () -> plan)); + Assertions.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + void boundsEntriesTotalRangesAndRangePayloadWithoutTruncatingResults() { + DeltaLakeScanPlanMemo oneEntry = new DeltaLakeScanPlanMemo(1, 10, 1 << 20); + Assertions.assertFalse(oneEntry.get(key, new ConnectorScanContext(), () -> plan).reused()); + for (int i = 0; i < 2; i++) { + Assertions.assertFalse(oneEntry.get(key("extra"), new ConnectorScanContext(), () -> plan).reused()); + } + DeltaLakeScanPlanMemo twoRanges = new DeltaLakeScanPlanMemo(10, 2, 1 << 20); + DeltaLakeScanPlanMemo.Plan oversized = new DeltaLakeScanPlanMemo.Plan( + List.of(range("a", 0), range("b", 0), range("c", 0)), null); + for (int i = 0; i < 2; i++) { + DeltaLakeScanPlanMemo.Outcome result = twoRanges.get(key, new ConnectorScanContext(), () -> oversized); + Assertions.assertFalse(result.reused()); + Assertions.assertEquals(3, result.plan().ranges().size()); + } + for (String table : List.of("first", "second")) { + Assertions.assertFalse(twoRanges.get(key(table), new ConnectorScanContext(), () -> plan).reused()); + } + Assertions.assertFalse(twoRanges.get(key, new ConnectorScanContext(), () -> plan).reused()); + Assertions.assertFalse(twoRanges.get(key, new ConnectorScanContext(), () -> plan).reused()); + + DeltaLakeScanPlanMemo bytes = new DeltaLakeScanPlanMemo(10, 100, 1024); + for (DeltaLakeScanRange range : List.of(range("a".repeat(2048), 0), range("a", 2048))) { + DeltaLakeScanPlanMemo.Plan largePayload = new DeltaLakeScanPlanMemo.Plan(List.of(range), null); + for (int i = 0; i < 2; i++) { + Assertions.assertFalse(bytes.get(key, new ConnectorScanContext(), () -> largePayload).reused()); + } + } + Assertions.assertFalse(bytes.get(key, new ConnectorScanContext(), () -> plan).reused()); + Assertions.assertTrue(bytes.get(key, new ConnectorScanContext(), () -> plan).reused()); + DeltaLakeScanPlanMemo.Plan empty = new DeltaLakeScanPlanMemo.Plan(List.of(), null); + Assertions.assertFalse(oneEntry.get(key("empty"), new ConnectorScanContext(), () -> empty).reused()); + Assertions.assertFalse(oneEntry.get(key("empty"), new ConnectorScanContext(), () -> empty).reused()); + } + + @Test + void distinguishesSnapshotCoordinatesAndTypedPredicatesEvenWhenTextMatches() { + DeltaLakeScanPlanMemo memo = new DeltaLakeScanPlanMemo(); + AtomicInteger loads = new AtomicInteger(); + Supplier loader = () -> { + loads.incrementAndGet(); + return plan; + }; + ConnectorLiteral bigint = new ConnectorLiteral(ConnectorType.of("BIGINT"), 1L); + ConnectorLiteral integer = new ConnectorLiteral(ConnectorType.of("INT"), 1L); + Assertions.assertEquals(bigint.toString(), integer.toString()); + List keys = List.of(key, + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "other-id", 0, + Optional.empty(), Set.of(), false, false, 128, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "id", 1, + Optional.empty(), Set.of(), false, false, 128, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "other-path", "id", 0, + Optional.empty(), Set.of(), false, false, 128, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "id", 0, + Optional.of(bigint), Set.of(), false, false, 128, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "id", 0, + Optional.of(integer), Set.of(), false, false, 128, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "id", 0, + Optional.empty(), Set.of(), false, false, 256, "UTC"), + new DeltaLakeScanPlanMemo.Key("u", "db", "table", "path", "id", 0, + Optional.empty(), Set.of(), false, false, 128, "Asia/Shanghai")); + for (DeltaLakeScanPlanMemo.Key item : keys) { + Assertions.assertFalse(memo.get(item, new ConnectorScanContext(), loader).reused()); + } + for (DeltaLakeScanPlanMemo.Key item : keys) { + Assertions.assertTrue(memo.get(item, new ConnectorScanContext(), loader).reused()); + } + Assertions.assertEquals(keys.size(), loads.get()); + } + + private static DeltaLakeScanPlanMemo.Key key(String table) { + return new DeltaLakeScanPlanMemo.Key("u", "db", table, "path", "id", 0, + Optional.empty(), Set.of(), false, false, 128, "UTC"); + } + + private static DeltaLakeScanRange range(String path, int inlineBytes) { + TDeltaLakeFileDesc descriptor = new TDeltaLakeFileDesc().setSchemaId(123); + descriptor.setDeletionVector(new TDeltaLakeDeletionVectorDesc().setInlineData(new byte[inlineBytes])); + return new DeltaLakeScanRange(path, 0, 10, 10, 1, 128, Map.of(), descriptor); + } + + private static void await(CountDownLatch latch) { + try { + Assertions.assertTrue(latch.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProviderTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProviderTest.java new file mode 100644 index 00000000000000..853b6ac68d8bf2 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeScanPlanProviderTest.java @@ -0,0 +1,660 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.ConnectorPushdownOps; +import org.apache.doris.connector.spi.ConnectorStatementScope; +import org.apache.doris.connector.spi.ConnectorStorageContext; +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.spi.pushdown.ConnectorComparison; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.scan.ConnectorSplitSource; +import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TDeltaLakeDeletionVectorDesc; +import org.apache.doris.thrift.TDeltaLakeFileDesc; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.stream.Collectors; + +class DeltaLakeScanPlanProviderTest { + @TempDir + Path warehouse; + + @Test + void identicalScansReuseRangesAndPublishMetricsOnEveryNode() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + appendStatsFile("b.parquet", "b", 11, 20); + TestingConnectorSession session = new TestingConnectorSession("reuse", + Map.of("enable_external_scan_task_reuse", "true", "file_split_size", "10")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider first = connector.getScanPlanProvider(); + List ranges = first.planScan(session, request(handle).build()); + Assertions.assertEquals(6, ranges.size()); + Assertions.assertThrows(UnsupportedOperationException.class, ranges::clear); + ConnectorScanPlanProvider second = connector.getScanPlanProvider(); + Assertions.assertSame(ranges, second.planScan(session, request(handle).build())); + assertMetrics(second, session, "2", "2", "2", "6", true); + Assertions.assertEquals("false", first.collectScanProfiles(session).get(0).getMetrics() + .get("PlanningReuseHit")); + Assertions.assertEquals("true", second.collectScanProfiles(session).get(0).getMetrics() + .get("PlanningReuseHit")); + StringBuilder explain = new StringBuilder(); + second.appendExplainInfo(explain, "", Map.of("deltalake.snapshot_version", "0")); + Assertions.assertTrue(explain.toString().contains("Delta files: 2/2, selected partitions: 2, splits: 6")); + + List count = second.planScan(session, request(handle).countPushdown(true).build()); + Assertions.assertEquals(20, count.get(0).getPushDownRowCount()); + Assertions.assertSame(count, first.planScan(session, request(handle).countPushdown(true).build())); + Assertions.assertEquals(-1, ranges.get(0).getPushDownRowCount()); + Assertions.assertTrue(second.planScan(session, request(handle).partitionsPrunedToEmpty(true).build()) + .isEmpty()); + List partition = second.planScan(session, + request(handle).requiredPartitions(List.of("p=a")).build()); + Assertions.assertEquals(3, partition.size()); + Assertions.assertSame(partition, first.planScan(session, + request(handle).requiredPartitions(List.of("p=a", "p=a")).build())); + for (long value : new long[] {5, 15, 5}) { + ConnectorComparison filter = new ConnectorComparison(ConnectorComparison.Operator.LT, + new ConnectorColumnRef("id", ConnectorType.of("BIGINT")), ConnectorLiteral.ofLong(value)); + List filtered = second.planScan(session, + request(handle).filter(Optional.of(filter)).build()); + Assertions.assertEquals(value == 5 ? 3 : 6, filtered.size()); + } + Assertions.assertEquals("true", second.collectScanProfiles(session).get(0).getMetrics() + .get("PlanningReuseHit")); + + ConnectorScanRequest cancelled = request(handle).build(); + cancelled.getScanContext().cancel(); + Assertions.assertThrows(CancellationException.class, () -> second.planScan(session, cancelled)); + Assertions.assertSame(ranges, second.planScan(session, request(handle).build())); + session.getStatementScope().endPlanning(); + Assertions.assertNotSame(ranges, first.planScan(session, request(handle).build())); + } + } + + @Test + void scanReuseStaysInsideItsQueryCatalogIdentityAndPlanningLifetime() throws Exception { + createTable(); + appendFile(0, "a.parquet", "a", 25); + Map enabled = Map.of("enable_external_scan_task_reuse", "true"); + TestingConnectorSession original = new TestingConnectorSession("scope", enabled); + ConnectorStatementScope shared = original.getStatementScope(); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(original)) { + ConnectorTableHandle handle = metadata.getTableHandle(original, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + List ranges = provider.planScan(original, request(handle).build()); + List independent = List.of( + new TestingConnectorSession("scope", enabled), + new TestingConnectorSession("scope"), + new TestingConnectorSession("scope", Map.of("enable_external_scan_task_reuse", "false")), + new TestingConnectorSession("scope", + Map.of("enable_external_scan_task_reuse", "true", "file_split_size", "10")) { + @Override + public ConnectorStatementScope getStatementScope() { + return shared; + } + }, + new TestingConnectorSession("scope", enabled) { + @Override + public ConnectorStatementScope getStatementScope() { + return shared; + } + + @Override + public String getTimeZone() { + return "Asia/Shanghai"; + } + }, + new TestingConnectorSession("scope", enabled) { + @Override + public ConnectorStatementScope getStatementScope() { + return ConnectorStatementScope.NONE; + } + }, + new TestingConnectorSession("another-query", enabled) { + @Override + public ConnectorStatementScope getStatementScope() { + return shared; + } + }, + new TestingConnectorSession("scope", enabled) { + @Override + public ConnectorStatementScope getStatementScope() { + return shared; + } + + @Override + public long getCatalogId() { + return 43; + } + }, + new TestingConnectorSession("scope", enabled) { + @Override + public ConnectorStatementScope getStatementScope() { + return shared; + } + + @Override + public String getUser() { + return "another-user"; + } + }); + for (TestingConnectorSession session : independent) { + Assertions.assertNotSame(ranges, provider.planScan(session, request(handle).build())); + Assertions.assertEquals("false", provider.collectScanProfiles(session).get(0).getMetrics() + .get("PlanningReuseHit")); + } + Assertions.assertNotSame(ranges, provider.planScan(null, request(handle).build())); + appendFile(1, "b.parquet", "b", 25); + connector.invalidateTable("db", "table"); + ConnectorTableHandle next = metadata.getTableHandle(new TestingConnectorSession("latest"), + "db", "table").orElseThrow(); + Assertions.assertEquals(2, provider.planScan(original, request(next).build()).size()); + Assertions.assertSame(ranges, provider.planScan(original, request(handle).build())); + shared.closeAll(); + Assertions.assertThrows(CancellationException.class, + () -> provider.planScan(original, request(handle).build())); + } + } + + @Test + void reusedGoldenDescriptorsCannotBeMutatedByAnotherScanNode() throws Exception { + Path resources = Path.of(getClass().getResource("/deltalake").toURI()).getParent(); + TestingConnectorSession session = new TestingConnectorSession("golden-reuse", + Map.of("enable_external_scan_task_reuse", "true")); + try (Connector connector = new DeltaLakeConnectorProvider().create( + Map.of("warehouse", resources.toString()), new RecordingConnectorContext()); + ConnectorMetadata metadata = connector.getMetadata(session)) { + for (String table : List.of("dv-partitioned-with-checkpoint", "v2-checkpoint-parquet")) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "deltalake", table).orElseThrow(); + List ranges = connector.getScanPlanProvider() + .planScan(session, request(handle).build()); + List reused = connector.getScanPlanProvider() + .planScan(session, request(handle).build()); + Assertions.assertSame(ranges, reused); + for (int i = 0; i < ranges.size(); i++) { + TTableFormatFileDesc expected = new TTableFormatFileDesc(); + TFileRangeDesc file = new TFileRangeDesc(); + ranges.get(i).populateRangeParams(expected, file); + TTableFormatFileDesc changed = expected.deepCopy(); + changed.getDeltalakeParams().setSchemaId(999); + changed.getDeltalakeParams().unsetDeletionVector(); + reused.get(i).populateRangeParams(changed, new TFileRangeDesc()); + Assertions.assertEquals(expected, changed); + } + } + } + } + + @Test + void scanProviderRejectsCastPredicatePushdown() throws Exception { + try (Connector connector = connector()) { + ConnectorPushdownOps pushdown = Assertions.assertInstanceOf( + ConnectorPushdownOps.class, connector.getScanPlanProvider()); + Assertions.assertFalse(pushdown.supportsCastPredicatePushdown(new TestingConnectorSession("cast"))); + } + } + + @Test + void streamingDecisionReusesCompleteStatementCountsAtThePinnedVersion() throws Exception { + createTable(); + appendFile(0, "first.parquet", "a", 257); + appendFile(0, "second.parquet", "b", 25); + TestingConnectorSession session = new TestingConnectorSession("known-files", Map.of("file_split_size", "128")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), false) >= 0); + metadata.listPartitions(session, handle, Optional.empty()); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(session, handle, Optional.empty(), false)); + // Few files need not mean few splits: keep normal byte-range planning for large files. + Assertions.assertEquals(4, provider.planScan(session, request(handle).build()).size()); + + TestingConnectorSession boundary = new TestingConnectorSession("boundary", + Map.of("num_files_in_batch_mode", "2")); + metadata.listPartitions(boundary, handle, Optional.empty()); + Assertions.assertEquals(2, provider.streamingSplitEstimate(boundary, handle, Optional.empty(), false)); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(new TestingConnectorSession("next-query"), + handle, Optional.empty(), false)); + + appendFile(1, "later.parquet", "c", 25); + connector.invalidateTable("db", "table"); + ConnectorTableHandle next = metadata.getTableHandle(new TestingConnectorSession("next"), + "db", "table").orElseThrow(); + Assertions.assertTrue(provider.streamingSplitEstimate(session, next, Optional.empty(), false) >= 0); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(session, handle, Optional.empty(), false)); + } + } + + @Test + void splitsByteRangesAndKeepsPartitionsAndFilesOnThePinnedVersion() throws Exception { + createTable(); + appendFile(0, "s3a://bucket/first.parquet", "a/b", 257); + appendFile(0, "s3a://bucket/null.parquet", null, 25); + appendFile(0, "s3a://bucket/sentinel.parquet", "__HIVE_DEFAULT_PARTITION__", 25); + TestingConnectorSession session = new TestingConnectorSession("first", Map.of("file_split_size", "128")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + appendFile(1, "s3a://bucket/later.parquet", "later", 25); + List partitions = metadata.listPartitions(session, handle, Optional.empty()); + Assertions.assertEquals(3, partitions.size()); + Assertions.assertEquals(3, partitions.stream().map(ConnectorPartitionInfo::getPartitionName) + .distinct().count()); + Assertions.assertEquals(1, partitions.stream() + .filter(partition -> partition.getPartitionValueNullFlags().get(0)).count()); + ConnectorScanPlanProvider planner = connector.getScanPlanProvider(); + List ranges = planner.planScan(session, request(handle).build()); + Assertions.assertEquals(5, ranges.size()); + Assertions.assertEquals(307, ranges.stream().mapToLong(ConnectorScanRange::getLength).sum()); + List first = ranges.stream() + .filter(range -> range.getPath().orElseThrow().equals("s3://bucket/first.parquet")) + .collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList(0L, 128L, 256L), + first.stream().map(ConnectorScanRange::getStart).collect(Collectors.toList())); + Assertions.assertEquals(3, planner.scannedPartitionCount(ranges).orElseThrow()); + Assertions.assertEquals(3, planner.planScan(session, request(handle) + .requiredPartitions(Collections.singletonList("p=a%2Fb")).build()).size()); + Assertions.assertTrue(planner.planScan(session, request(handle).partitionsPrunedToEmpty(true).build()) + .isEmpty()); + Map properties = planner.getScanNodeProperties(session, handle, + Collections.emptyList(), Optional.empty()); + Assertions.assertEquals("parquet", properties.get(ScanNodePropertyKeys.FILE_FORMAT_TYPE)); + Assertions.assertEquals("p", properties.get(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + Assertions.assertEquals("test-key", properties.get(ScanNodePropertyKeys.LOCATION_PREFIX + "AWS_ACCESS_KEY")); + connector.invalidateTable("db", "table"); + TestingConnectorSession next = new TestingConnectorSession("next"); + ConnectorTableHandle latest = metadata.getTableHandle(next, "db", "table").orElseThrow(); + Assertions.assertEquals(4, metadata.listPartitions(next, latest, Optional.empty()).size()); + } + } + + @Test + void emptyTableReturnsNoSplitsAndInvalidFilesFailPlanning() throws Exception { + createTable(); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(null)) { + ConnectorTableHandle empty = metadata.getTableHandle(null, "db", "table").orElseThrow(); + Assertions.assertTrue(connector.getScanPlanProvider().planScan(null, request(empty).build()).isEmpty()); + appendFile(1, "broken.parquet", "a", 0); + ConnectorTableHandle invalid = metadata.getTableHandle(null, "db", "table").orElseThrow(); + Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getScanPlanProvider().planScan(null, request(invalid).build())); + } + } + + @Test + void rangeThriftKeepsNullEmptyAndSentinelValuesDistinctAndOwnsItsDescriptor() { + TDeltaLakeFileDesc descriptor = new TDeltaLakeFileDesc(); + descriptor.setDeletionVector(new TDeltaLakeDeletionVectorDesc().setSizeInBytes(12).setCardinality(1) + .setInlineData(new byte[] {1, 2, 3})); + for (String value : Arrays.asList(null, "", "__HIVE_DEFAULT_PARTITION__")) { + DeltaLakeScanRange range = new DeltaLakeScanRange("s3://bucket/data.parquet", 0, 10, 10, 1, 128, + Collections.singletonMap("p", value), descriptor); + TFileRangeDesc file = new TFileRangeDesc(); + file.setColumnsFromPath(Collections.singletonList("wrong generic path value")); + TTableFormatFileDesc format = new TTableFormatFileDesc(); + range.populateRangeParams(format, file); + Assertions.assertEquals(TFileFormatType.FORMAT_PARQUET, file.getFormatType()); + Assertions.assertEquals(Collections.singletonList("p"), file.getColumnsFromPathKeys()); + Assertions.assertEquals(Collections.singletonList(value == null ? "" : value), file.getColumnsFromPath()); + Assertions.assertEquals(Collections.singletonList(value == null), file.getColumnsFromPathIsNull()); + Assertions.assertFalse(format.getDeltalakeParams().isSetSchemaId()); + format.getDeltalakeParams().getDeletionVector().setCardinality(999); + range.populateRangeParams(format, file); + Assertions.assertEquals(1, format.getDeltalakeParams().getDeletionVector().getCardinality()); + } + } + + @Test + void skipsFilesByStatisticsAndCountsOnlyCompleteUnfilteredMetadata() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + appendStatsFile("b.parquet", "b", 11, 20); + TestingConnectorSession session = new TestingConnectorSession("stats"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + ConnectorComparison filter = new ConnectorComparison(ConnectorComparison.Operator.LT, + new ConnectorColumnRef("id", ConnectorType.of("BIGINT")), ConnectorLiteral.ofLong(5)); + List filtered = provider.planScan(session, request(handle) + .filter(Optional.of(filter)).build()); + Assertions.assertEquals(1, filtered.size()); + Assertions.assertTrue(filtered.get(0).getPath().orElseThrow().endsWith("a.parquet")); + Assertions.assertFalse(provider.canServeMetadataOnlyCount(session, handle, Optional.of(filter))); + Assertions.assertTrue(provider.canServeMetadataOnlyCount(session, handle, Optional.empty())); + List count = provider.planScan(session, request(handle).countPushdown(true).build()); + Assertions.assertEquals(1, count.size()); + Assertions.assertEquals(20, count.get(0).getPushDownRowCount()); + TTableFormatFileDesc format = new TTableFormatFileDesc(); + count.get(0).populateRangeParams(format, new TFileRangeDesc()); + Assertions.assertEquals(20, format.getTableLevelRowCount()); + Assertions.assertTrue(count.get(0).isNativeReadRange()); + Assertions.assertEquals(-1, provider.planScan(session, request(handle).countPushdown(true) + .filter(Optional.of(filter)).build()).get(0).getPushDownRowCount()); + } + } + + @Test + void missingStatisticsFallBackToAllFilesWithoutGuessingCounts() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + appendFile(0, "b.parquet", "b", 25); + TestingConnectorSession session = new TestingConnectorSession("missing-stats"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertFalse(provider.canServeMetadataOnlyCount(session, handle, Optional.empty())); + List ranges = provider.planScan(session, request(handle).countPushdown(true).build()); + Assertions.assertEquals(2, ranges.size()); + Assertions.assertTrue(ranges.stream().allMatch(range -> range.getPushDownRowCount() == -1)); + } + } + + @Test + void streamingMatchesSynchronousByteRangesAndKeepsThePinnedVersion() throws Exception { + createTable(); + appendFile(0, "a.parquet", "a", 257); + appendFile(0, "b.parquet", null, 25); + TestingConnectorSession session = new TestingConnectorSession("stream", Map.of("file_split_size", "128")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), false) >= 0); + try (ConnectorSplitSource source = provider.streamSplits(session, request(handle).build())) { + appendFile(1, "later.parquet", "later", 25); + List streamed = new ArrayList<>(); + while (source.hasNext()) { + streamed.add(source.next()); + } + List synchronous = provider.planScan(session, request(handle).build()); + Assertions.assertEquals(synchronous.size(), streamed.size()); + for (int i = 0; i < streamed.size(); i++) { + Assertions.assertEquals(synchronous.get(i).getPath(), streamed.get(i).getPath()); + Assertions.assertEquals(synchronous.get(i).getStart(), streamed.get(i).getStart()); + Assertions.assertEquals(synchronous.get(i).getLength(), streamed.get(i).getLength()); + Assertions.assertEquals(synchronous.get(i).getPartitionValues(), streamed.get(i).getPartitionValues()); + } + Assertions.assertEquals(4, streamed.size()); + Assertions.assertFalse(source.hasNext()); + } + } + } + + @Test + void streamingCountCollapsesOnlyCompleteStatistics() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + TestingConnectorSession session = new TestingConnectorSession("stream-count"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), true) >= 0); + try (ConnectorSplitSource source = provider.streamSplits(session, request(handle).build())) { + Assertions.assertTrue(source.hasNext()); + Assertions.assertEquals(10, source.next().getPushDownRowCount()); + Assertions.assertFalse(source.hasNext()); + } + appendFile(1, "b.parquet", "b", 25); + connector.invalidateTable("db", "table"); + TestingConnectorSession next = new TestingConnectorSession("stream-no-stats"); + ConnectorTableHandle newer = metadata.getTableHandle(next, "db", "table").orElseThrow(); + Assertions.assertTrue(provider.streamingSplitEstimate(next, newer, Optional.empty(), true) >= 0); + try (ConnectorSplitSource source = provider.streamSplits(next, request(newer).build())) { + int files = 0; + while (source.hasNext()) { + Assertions.assertEquals(-1, source.next().getPushDownRowCount()); + files++; + } + Assertions.assertEquals(2, files); + } + } + } + + @Test + void streamingHonorsRequestPartitionsAndCancellation() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + appendStatsFile("b.parquet", "b", 11, 20); + TestingConnectorSession session = new TestingConnectorSession("stream-request"); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), true) >= 0); + ConnectorScanRequest subset = request(handle).requiredPartitions(List.of("p=a")).build(); + try (ConnectorSplitSource source = provider.streamSplits(session, subset)) { + Assertions.assertTrue(source.hasNext()); + ConnectorScanRange range = source.next(); + Assertions.assertEquals("a", range.getPartitionValues().get("p")); + Assertions.assertEquals(-1, range.getPushDownRowCount()); + Assertions.assertFalse(source.hasNext()); + } + try (ConnectorSplitSource source = provider.streamSplits(session, + request(handle).partitionsPrunedToEmpty(true).build())) { + Assertions.assertFalse(source.hasNext()); + } + ConnectorScanRequest cancelled = request(handle).build(); + try (ConnectorSplitSource source = provider.streamSplits(session, cancelled)) { + cancelled.getScanContext().cancel(); + Assertions.assertThrows(CancellationException.class, source::hasNext); + Assertions.assertFalse(source.hasNext(), "cancelled consumption closes its source"); + } + Assertions.assertThrows(CancellationException.class, + () -> provider.streamSplits(session, cancelled)); + } + } + + @Test + void streamingDefersReadingUntilConsumptionAndHonorsTheBatchToggle() throws Exception { + createTable(); + TestingConnectorSession session = new TestingConnectorSession("lazy", + Map.of("enable_external_table_batch_mode", "false")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(session, handle, Optional.empty(), false)); + ConnectorSplitSource source = provider.streamSplits(session, request(handle).build()); + Files.delete(warehouse.resolve("db/table/_delta_log/00000000000000000000.json")); + source.close(); + Assertions.assertFalse(source.hasNext()); + } + } + + @Test + void profilesDistinguishFilesSplitsAndPartitionsAndKeepCountMetadata() throws Exception { + createTable(); + appendStatsFile("a.parquet", "a", 1, 10); + appendStatsFile("b.parquet", "b", 11, 20); + TestingConnectorSession session = new TestingConnectorSession("metrics", Map.of("file_split_size", "10")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + ConnectorComparison dataFilter = new ConnectorComparison(ConnectorComparison.Operator.LT, + new ConnectorColumnRef("id", ConnectorType.of("BIGINT")), ConnectorLiteral.ofLong(5)); + provider.planScan(session, request(handle).filter(Optional.of(dataFilter)).build()); + // No CRC or prior partition listing: the exhaustive Kernel report supplies the total. + assertMetrics(provider, session, "2", "1", "1", "3", true); + StringBuilder explain = new StringBuilder(); + provider.appendExplainInfo(explain, "", provider.getScanNodeProperties(session, handle, + Collections.emptyList(), Optional.of(dataFilter))); + Assertions.assertTrue(explain.toString().contains("Delta files: 1/2, selected partitions: 1, splits: 3")); + provider.planScan(session, request(handle).countPushdown(true).build()); + assertMetrics(provider, session, "2", "2", "2", "1", true); + // A different producer reuses statement COUNT facts, not the representative range's partition. + ConnectorScanPlanProvider another = connector.getScanPlanProvider(); + another.planScan(session, request(handle).countPushdown(true).build()); + assertMetrics(another, session, "2", "2", "2", "1", true); + } + } + + @Test + void streamingProfilesPublishAfterCloseAndIdentifyPartialEnumeration() throws Exception { + createTable(); + appendFile(0, "a.parquet", "a", 25); + appendFile(0, "b.parquet", "b", 25); + TestingConnectorSession session = new TestingConnectorSession("stream-metrics", + Map.of("file_split_size", "10")); + try (Connector connector = connector(); ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + metadata.listPartitions(session, handle, Optional.empty()); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + ConnectorComparison filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("p", ConnectorType.of("STRING")), ConnectorLiteral.ofString("a")); + try (ConnectorSplitSource source = provider.streamSplits(session, + request(handle).filter(Optional.of(filter)).build())) { + Assertions.assertTrue(provider.collectScanProfiles(session).isEmpty()); + source.next(); + } + assertMetrics(provider, session, "2", "1", "1", "1", false); + Assertions.assertTrue(provider.streamingSplitEstimate(new TestingConnectorSession("after-partial"), + handle, Optional.empty(), false) >= 0); + try (ConnectorSplitSource source = provider.streamSplits(session, + request(handle).filter(Optional.of(filter)).build())) { + while (source.hasNext()) { + source.next(); + } + } + // The partition predicate's replay counter must not replace the complete table total. + assertMetrics(provider, session, "2", "1", "1", "3", true); + Assertions.assertTrue(provider.streamingSplitEstimate(new TestingConnectorSession("after-partition-pruning"), + handle, Optional.empty(), false) >= 0); + provider.planScan(session, request(handle).partitionsPrunedToEmpty(true).build()); + assertMetrics(provider, session, "2", "0", "0", "0", true); + Assertions.assertTrue(provider.streamingSplitEstimate(new TestingConnectorSession("after-empty-request"), + handle, Optional.empty(), false) >= 0); + provider.planScan(session, request(handle).build()); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(new TestingConnectorSession("after-full-scan"), + handle, Optional.empty(), false)); + Assertions.assertTrue(DeltaLakeStatementScope.FILE_COUNT_NAMESPACE.startsWith("deltalake.")); + } + } + + @Test + void checkpointPartitionPruningDoesNotTurnTheReplayCounterIntoATableTotal() throws Exception { + Path resources = Path.of(getClass().getResource("/deltalake").toURI()).getParent(); + TestingConnectorSession session = new TestingConnectorSession("checkpoint-metrics"); + try (Connector connector = new DeltaLakeConnectorProvider().create( + Map.of("warehouse", resources.toString()), new RecordingConnectorContext()); + ConnectorMetadata metadata = connector.getMetadata(session)) { + ConnectorTableHandle handle = metadata.getTableHandle(session, + "deltalake", "dv-partitioned-with-checkpoint").orElseThrow(); + metadata.listPartitions(session, handle, Optional.empty()); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + ConnectorComparison filter = new ConnectorComparison(ConnectorComparison.Operator.EQ, + new ConnectorColumnRef("part", ConnectorType.of("INT")), ConnectorLiteral.ofLong(1)); + List ranges = provider.planScan(session, + request(handle).filter(Optional.of(filter)).build()); + Assertions.assertFalse(ranges.isEmpty()); + Assertions.assertTrue(ranges.size() < 15); + Map metrics = provider.collectScanProfiles(session).get(0).getMetrics(); + Assertions.assertEquals("15", metrics.get("TotalFiles")); + Assertions.assertEquals(Integer.toString(ranges.size()), metrics.get("SelectedFiles")); + Assertions.assertEquals("1", metrics.get("SelectedPartitions")); + } + } + + private static void assertMetrics(ConnectorScanPlanProvider provider, TestingConnectorSession session, + String total, String selected, String partitions, String splits, boolean complete) { + Map metrics = provider.collectScanProfiles(session).get(0).getMetrics(); + Assertions.assertEquals("0", metrics.get("SnapshotVersion")); + Assertions.assertEquals(total, metrics.get("TotalFiles")); + Assertions.assertEquals(selected, metrics.get("SelectedFiles")); + Assertions.assertEquals(partitions, metrics.get("SelectedPartitions")); + Assertions.assertEquals(splits, metrics.get("GeneratedSplits")); + Assertions.assertEquals(Boolean.toString(complete), metrics.get("EnumerationComplete")); + } + + private void appendStatsFile(String path, String partition, long minimum, long maximum) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + String stats = mapper.writeValueAsString(Map.of("numRecords", maximum - minimum + 1, + "minValues", Map.of("id", minimum), "maxValues", Map.of("id", maximum), + "nullCount", Map.of("id", 0))); + String action = mapper.writeValueAsString(Map.of("add", Map.of("path", path, "size", 25, + "partitionValues", Map.of("p", partition), "stats", stats, + "modificationTime", 1L, "dataChange", true))); + Files.writeString(warehouse.resolve("db/table/_delta_log/00000000000000000000.json"), + action + "\n", StandardOpenOption.APPEND); + } + + private void createTable() throws Exception { + DeltaLakeTestTables.commit(warehouse.resolve("db/table"), 0, "table-id", + new StructType().add("id", LongType.LONG).add("p", StringType.STRING), + Collections.singletonList("p")); + } + + private void appendFile(long version, String path, String partition, long size) throws Exception { + String action = new ObjectMapper().writeValueAsString(Map.of("add", Map.of("path", path, "size", size, + "partitionValues", Collections.singletonMap("p", partition), + "modificationTime", 1L, "dataChange", true))); + Files.writeString(warehouse.resolve("db/table/_delta_log").resolve(String.format("%020d.json", version)), + action + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } + + private static ConnectorScanRequest.Builder request(ConnectorTableHandle handle) { + return ConnectorScanRequest.builder(handle, Collections.emptyList()); + } + + private Connector connector() { + return new DeltaLakeConnectorProvider().create(Collections.singletonMap("warehouse", warehouse.toString()), + new RecordingConnectorContext() { + @Override + public ConnectorStorageContext getStorageContext() { + return new ConnectorStorageContext() { + @Override + public String normalizeStorageUri(String uri) { + return uri.replace("s3a://", "s3://"); + } + + @Override + public Map getBackendStorageProperties() { + return Map.of("AWS_ACCESS_KEY", "test-key"); + } + }; + } + }); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilderTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilderTest.java new file mode 100644 index 00000000000000..a05fead0017fb3 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSchemaBuilderTest.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TSchema; + +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.FieldMetadata; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DeltaLakeSchemaBuilderTest { + @Test + void buildsLogicalAndPhysicalNamesWithStableNestedIds() { + StructType nested = new StructType().add(field("Renamed", LongType.LONG, 2, "physical-child")); + StructType schema = new StructType().add(field("Root", nested, 1, "physical-root")); + TSchema logical = DeltaLakeSchemaBuilder.mappingSchema(schema, false); + TSchema physical = DeltaLakeSchemaBuilder.mappingSchema(schema, true); + TField logicalRoot = logical.getRootField().getFields().get(0).getFieldPtr(); + TField physicalRoot = physical.getRootField().getFields().get(0).getFieldPtr(); + Assertions.assertEquals(0, logical.getSchemaId()); + Assertions.assertEquals(1, physical.getSchemaId()); + Assertions.assertEquals("root", logicalRoot.getName()); + Assertions.assertEquals("physical-root", physicalRoot.getName()); + Assertions.assertEquals(1, logicalRoot.getId()); + TField child = logicalRoot.getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assertions.assertEquals("renamed", child.getName()); + Assertions.assertEquals(2, child.getId()); + Assertions.assertEquals("physical-child", physicalRoot.getNestedField().getStructField() + .getFields().get(0).getFieldPtr().getName()); + } + + @Test + void retainsOptionalCollectionIdsFromTheNearestStructField() { + FieldMetadata ids = FieldMetadata.builder().putLong("col-map.key", 11) + .putLong("col-map.value", 12).putLong("col-map.value.element", 13).build(); + StructField mapped = field("Map", new MapType(StringType.STRING, + new ArrayType(LongType.LONG, true), true), 1, "col-map"); + mapped = mapped.withNewMetadata(FieldMetadata.builder().fromMetadata(mapped.getMetadata()) + .putFieldMetadata("delta.columnMapping.nested.ids", ids).build()); + for (boolean physical : new boolean[] {false, true}) { + TField root = DeltaLakeSchemaBuilder.mappingSchema(new StructType().add(mapped), physical) + .getRootField().getFields().get(0).getFieldPtr(); + Assertions.assertEquals(11, root.getNestedField().getMapField().getKeyField().getFieldPtr().getId()); + TField value = root.getNestedField().getMapField().getValueField().getFieldPtr(); + Assertions.assertEquals(12, value.getId()); + Assertions.assertEquals(13, value.getNestedField().getArrayField().getItemField().getFieldPtr().getId()); + } + StructField plain = field("Array", new ArrayType(LongType.LONG, true), 3, "col-array"); + TField array = DeltaLakeSchemaBuilder.mappingSchema(new StructType().add(plain), true) + .getRootField().getFields().get(0).getFieldPtr(); + Assertions.assertFalse(array.getNestedField().getArrayField().getItemField().getFieldPtr().isSetId()); + } + + @Test + void serializesBothSchemasAndTimestampSemanticsWithoutChangingNoneMode() { + StructType schema = new StructType().add(field("TS", TimestampType.TIMESTAMP, 1, "col-ts")) + .add(field("NTZ", TimestampNTZType.TIMESTAMP_NTZ, 2, "col-ntz")); + TFileScanRangeParams params = new TFileScanRangeParams(); + DeltaLakeSchemaBuilder.applyMapping(params, null); + Assertions.assertFalse(params.isSetCurrentSchemaId()); + DeltaLakeSchemaBuilder.applyMapping(params, DeltaLakeSchemaBuilder.encodeMapping(schema)); + Assertions.assertEquals(0, params.getCurrentSchemaId()); + Assertions.assertEquals(2, params.getHistorySchemaInfoSize()); + for (TSchema entry : params.getHistorySchemaInfo()) { + Assertions.assertTrue(entry.getRootField().getFields().get(0).getFieldPtr().isTimestampIsAdjustedToUtc()); + Assertions.assertFalse(entry.getRootField().getFields().get(1).getFieldPtr().isTimestampIsAdjustedToUtc()); + } + Assertions.assertThrows(DorisConnectorException.class, () -> DeltaLakeSchemaBuilder.applyMapping(params, "!")); + } + + private static StructField field(String name, io.delta.kernel.types.DataType type, long id, String physical) { + return new StructField(name, type, true, FieldMetadata.builder().putLong("delta.columnMapping.id", id) + .putString("delta.columnMapping.physicalName", physical).build()); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCacheTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCacheTest.java new file mode 100644 index 00000000000000..7aa7a4ef990145 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotCacheTest.java @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.CatalogMetaCache; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.data.Row; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.utils.CloseableIterator; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +class DeltaLakeSnapshotCacheTest { + @TempDir + Path table; + + private final Engine engine = DefaultEngine.create(new Configuration()); + private final DeltaLakeSnapshotCache.TableKey key = + new DeltaLakeSnapshotCache.TableKey("alice", "db", "table", "/table"); + + @Test + void fixedTtlStartsAfterLoadingAndHitsNeverExtendIt() throws Exception { + Snapshot first = snapshot(0); + Snapshot second = snapshot(1); + AtomicLong clock = new AtomicLong(); + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, Map.of(), clock::get); + Supplier loader = () -> { + clock.addAndGet(TimeUnit.SECONDS.toNanos(5)); + return loads.incrementAndGet() == 1 ? first : second; + }; + Assertions.assertSame(first, cache.latest(key, loader, version -> Assertions.fail("snapshot missed"))); + for (int seconds = 6; seconds < 35; seconds++) { + clock.set(TimeUnit.SECONDS.toNanos(seconds)); + Assertions.assertSame(first, cache.latest(key, loader, version -> Assertions.fail("snapshot missed"))); + } + Assertions.assertEquals(1, loads.get()); + clock.set(TimeUnit.SECONDS.toNanos(35)); + Assertions.assertSame(second, cache.latest(key, loader, version -> Assertions.fail("snapshot missed"))); + Assertions.assertEquals(2, loads.get()); + Assertions.assertEquals(0, first.getVersion()); + } + } + + @Test + void disabledLatestAlwaysLoadsAndARejectedSnapshotDoesNotDoubleLoadTheMiss() throws Exception { + Snapshot snapshot = snapshot(0); + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, + Map.of("meta.cache.deltalake.latest.ttl-second", "0", + "meta.cache.deltalake.snapshot.max-weight", "1")); + for (int i = 0; i < 3; i++) { + Assertions.assertSame(snapshot, cache.latest(key, () -> { + loads.incrementAndGet(); + return snapshot; + }, version -> Assertions.fail("duplicate load"))); + } + Assertions.assertEquals(3, loads.get()); + } + } + + @Test + void usersScopesAndWeightAdmissionRemainIndependent() throws Exception { + Snapshot first = snapshot(0); + Snapshot second = snapshot(1); + AtomicInteger versions = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, + Map.of("meta.cache.deltalake.snapshot.max-weight", "1")); + Assertions.assertSame(first, cache.latest(key, () -> first, version -> Assertions.fail())); + Assertions.assertSame(first, cache.latest(key, () -> Assertions.fail("latest missed"), version -> { + versions.incrementAndGet(); + Assertions.assertEquals(0, version); + return first; + })); + Assertions.assertEquals(1, versions.get()); + DeltaLakeSnapshotCache.TableKey bob = new DeltaLakeSnapshotCache.TableKey("bob", "db", "table", "/table"); + Assertions.assertSame(second, cache.latest(bob, () -> second, version -> Assertions.fail())); + owner.invalidateTable("db", "table"); + Assertions.assertSame(second, cache.latest(key, () -> second, version -> Assertions.fail())); + owner.invalidateDatabase("db"); + Assertions.assertSame(first, cache.latest(key, () -> first, version -> Assertions.fail())); + owner.invalidateCatalog(); + Assertions.assertSame(second, cache.latest(key, () -> second, version -> Assertions.fail())); + } + } + + @Test + void refreshFencesAConcurrentLoadWithoutChangingItsStatementsSnapshot() throws Exception { + Snapshot first = snapshot(0); + Snapshot second = snapshot(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch loading = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, Map.of()); + Future pending = executor.submit(() -> cache.latest(key, () -> { + loading.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return first; + }, version -> Assertions.fail())); + Assertions.assertTrue(loading.await(10, TimeUnit.SECONDS)); + owner.invalidateTable("db", "table"); + Assertions.assertSame(second, cache.latest(key, () -> second, version -> Assertions.fail())); + finish.countDown(); + Assertions.assertSame(first, pending.get(10, TimeUnit.SECONDS)); + Assertions.assertSame(second, cache.latest(key, () -> Assertions.fail("stale load republished"), + version -> Assertions.fail())); + } finally { + finish.countDown(); + executor.shutdownNow(); + } + } + + @Test + void weightBoundLimitsAdmissionButKeepsTheVersionPin() throws Exception { + Snapshot snapshot = snapshot(0); + long bytes = DeltaKernelCompat.snapshotRetainedBytes(snapshot); + AtomicInteger reloads = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, + Map.of("meta.cache.deltalake.snapshot.max-weight", String.valueOf(bytes + 2048))); + cache.latest(key, () -> snapshot, version -> Assertions.fail()); + DeltaLakeSnapshotCache.TableKey another = + new DeltaLakeSnapshotCache.TableKey("alice", "db", "another", "/another"); + cache.latest(another, () -> snapshot, version -> Assertions.fail()); + cache.latest(key, () -> Assertions.fail("latest pin evicted"), version -> { + reloads.incrementAndGet(); + return snapshot; + }); + cache.latest(another, () -> Assertions.fail("latest pin evicted"), version -> { + reloads.incrementAndGet(); + return snapshot; + }); + Assertions.assertEquals(1, reloads.get()); + Assertions.assertEquals(1, owner.entries().get("deltalake-snapshot").size()); + } + } + + @Test + void concurrentScansOfOneCachedCheckpointSnapshotHaveIndependentIterators() throws Exception { + Path fixture = Path.of(getClass().getResource("/deltalake/dv-partitioned-with-checkpoint").toURI()); + ExecutorService executor = Executors.newFixedThreadPool(8); + CyclicBarrier opened = new CyclicBarrier(8); + try (DeltaLakeFileIO io = new DeltaLakeFileIO(new Configuration(), new RecordingConnectorContext()); + CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + Engine sharedEngine = DefaultEngine.create(io); + DeltaLakeSnapshotCache cache = new DeltaLakeSnapshotCache(owner, Map.of()); + Snapshot cached = cache.latest(key, () -> TableManager.loadSnapshot(fixture.toString()).build(sharedEngine), + version -> Assertions.fail("unexpected reload")); + List> scans = new ArrayList<>(); + for (int worker = 0; worker < 8; worker++) { + boolean earlyClose = worker % 2 == 0; + scans.add(executor.submit(() -> { + for (int iteration = 0; iteration < 3; iteration++) { + Snapshot shared = cache.latest(key, () -> Assertions.fail("latest missed"), + version -> Assertions.fail("snapshot missed")); + Assertions.assertSame(cached, shared); + try (CloseableIterator files = DeltaLakeScanFiles.open( + shared, sharedEngine, !earlyClose, Optional.empty())) { + Assertions.assertTrue(files.hasNext()); + files.next(); + // All scans have opened and consumed a row before half close their own iterators. + opened.await(10, TimeUnit.SECONDS); + if (!earlyClose) { + int count = 1; + while (files.hasNext()) { + files.next(); + count++; + } + Assertions.assertEquals(15, count); + } + } + Assertions.assertEquals(35, + DeltaLakeStatisticsCache.collect(shared, sharedEngine).tableStatistics().getRowCount()); + } + return null; + })); + } + for (Future scan : scans) { + scan.get(30, TimeUnit.SECONDS); + } + } finally { + executor.shutdownNow(); + } + } + + private Snapshot snapshot(long version) throws Exception { + DeltaLakeTestTables.commit(table, version, "cached-table", new StructType().add("id", LongType.LONG)); + return TableManager.loadSnapshot(table.toString()).atVersion(version).build(engine); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotPinTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotPinTest.java new file mode 100644 index 00000000000000..9d4f128780d5ef --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeSnapshotPinTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.file.Path; +import java.util.Collections; + +class DeltaLakeSnapshotPinTest { + @TempDir + Path table; + + private final Engine engine = DefaultEngine.create(new Configuration()); + + @Test + void retainsSchemaAndVersionAcrossNewCommitsAndSerialization() throws Exception { + StructType original = new StructType().add("ID", LongType.LONG, true); + commit(0, "original", original); + Snapshot first = TableManager.loadSnapshot(table.toString()).build(engine); + DeltaLakeTableHandle handle = new DeltaLakeTableHandle("db", "table", first); + commit(1, "original", original.add("ADDED", StringType.STRING, true)); + Snapshot latest = TableManager.loadSnapshot(table.toString()).build(engine); + Assertions.assertEquals(1, latest.getVersion()); + Assertions.assertEquals(2, latest.getSchema().length()); + Assertions.assertSame(first, handle.snapshot(engine)); + + DeltaLakeTableHandle restored = roundTrip(handle); + Assertions.assertEquals(0, restored.snapshot(engine).getVersion()); + Assertions.assertEquals(1, restored.snapshot(engine).getSchema().length()); + DeltaLakeCatalogProperties properties = + DeltaLakeCatalogProperties.of(Collections.singletonMap("warehouse", table.toString())); + ConnectorTableSchema schema = DeltaLakeSchemaBuilder.build("table", restored.snapshot(engine), properties); + Assertions.assertEquals("ID", schema.getColumns().get(0).getName()); + Assertions.assertEquals("BIGINT", schema.getColumns().get(0).getType().getTypeName()); + Assertions.assertEquals(first.getPath(), schema.getProperties().get(ConnectorTableSchema.SHOW_LOCATION_KEY)); + } + + @Test + void deserializedHandleRejectsRecreationAtSamePathAndVersion() throws Exception { + commit(0, "original", new StructType().add("id", LongType.LONG)); + DeltaLakeTableHandle handle = roundTrip(new DeltaLakeTableHandle("db", "table", + TableManager.loadSnapshot(table.toString()).build(engine))); + commit(0, "replacement", new StructType().add("id", LongType.LONG)); + Assertions.assertThrows(DorisConnectorException.class, () -> handle.snapshot(engine)); + } + + private void commit(long version, String id, StructType schema) throws Exception { + DeltaLakeTestTables.commit(table, version, id, schema); + } + + private DeltaLakeTableHandle roundTrip(DeltaLakeTableHandle handle) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(handle); + } + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (DeltaLakeTableHandle) in.readObject(); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCacheTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCacheTest.java new file mode 100644 index 00000000000000..f743497a136708 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeStatisticsCacheTest.java @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.deltalake.DeltaLakeSnapshotCache.SnapshotKey; +import org.apache.doris.connector.deltalake.DeltaLakeSnapshotCache.TableKey; +import org.apache.doris.connector.deltalake.DeltaLakeStatisticsCache.Statistics; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.kernel.Snapshot; +import io.delta.kernel.TableManager; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.StructType; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.LongConsumer; + +@Timeout(30) +class DeltaLakeStatisticsCacheTest { + @TempDir + Path warehouse; + private final Engine engine = DefaultEngine.create(new Configuration()); + private final SnapshotKey key = key("alice", "table-id", 0); + private final ConnectorTableStatistics exact = new ConnectorTableStatistics(30, 300); + private final Statistics collected = new Statistics(exact, 2); + private final Optional fallback = Optional.of(new ConnectorTableStatistics(-1, 300)); + + @Test + void coldLookupReturnsImmediatelyAndConcurrentLookupsShareOneLoad() throws Exception { + QueueExecutor executor = new QueueExecutor(); + ExecutorService callers = Executors.newFixedThreadPool(8); + CyclicBarrier start = new CyclicBarrier(8); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + Assertions.assertTrue(executor.tasks.isEmpty(), "a planning hint never submits statistics work"); + List> requests = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + requests.add(callers.submit(() -> { + start.await(10, TimeUnit.SECONDS); + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> collected)); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty(), "do not wait for the queued load"); + return null; + })); + } + for (Future request : requests) { + request.get(10, TimeUnit.SECONDS); + } + Assertions.assertEquals(1, executor.tasks.size()); + executor.runNext(); + Assertions.assertEquals(Optional.of(exact), cache.get(key, fallback, () -> Assertions.fail("reloaded"))); + Assertions.assertEquals(2, cache.cachedFileCount(key).orElseThrow()); + } finally { + callers.shutdownNow(); + } + } + + @Test + void isolatesUsersVersionsTableIdentityAndPath() { + QueueExecutor executor = new QueueExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + List keys = List.of(key, key("bob", "table-id", 0), key("alice", "table-id", 1), + key("alice", "replacement-id", 0), + new SnapshotKey(new TableKey("alice", "db", "table", "/moved"), "table-id", 0)); + for (int i = 0; i < keys.size(); i++) { + ConnectorTableStatistics value = new ConnectorTableStatistics(i, 100); + long files = i + 1; + Assertions.assertEquals(fallback, cache.get(keys.get(i), fallback, () -> new Statistics(value, files))); + executor.runNext(); + } + for (int i = 0; i < keys.size(); i++) { + Assertions.assertEquals(i, cache.get(keys.get(i), fallback, () -> Assertions.fail()).orElseThrow() + .getRowCount()); + Assertions.assertEquals(i + 1, cache.cachedFileCount(keys.get(i)).orElseThrow()); + } + owner.invalidateDatabase("db"); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> collected)); + } + } + + @Test + void refreshAndCloseCancelQueuedWork() { + QueueExecutor executor = new QueueExecutor(); + AtomicInteger loads = new AtomicInteger(); + CatalogMetaCache owner = CatalogMetaCache.unmanaged(); + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + Callable loader = () -> { + loads.incrementAndGet(); + return collected; + }; + try { + cache.get(key, fallback, loader); + owner.invalidateTable("db", "table"); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + cache.get(key, fallback, loader); + executor.runNext(); // The pre-REFRESH task was cancelled before it opened storage. + Assertions.assertEquals(0, loads.get()); + executor.runNext(); + Assertions.assertEquals(1, loads.get()); + owner.invalidateCatalog(); + cache.get(key, fallback, loader); + } finally { + owner.close(); + } + executor.runNext(); + Assertions.assertEquals(1, loads.get()); + } + + @Test + void lateCompletionAfterRefreshCannotReplaceNewStatistics() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + cache.get(key, fallback, () -> { + started.countDown(); + // Model remote IO that finishes even after the caller cancels it. + boolean waiting = true; + while (waiting) { + try { + waiting = !release.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + // Deliberate: the result still must not reappear after REFRESH. + } + } + finished.countDown(); + return new Statistics(new ConnectorTableStatistics(999, 999), 999); + }); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty(), "do not wait for running IO"); + owner.invalidateTable("db", "table"); + Assertions.assertEquals(exact, await(() -> cache.get(key, fallback, () -> collected), 30)); + release.countDown(); + Assertions.assertTrue(finished.await(10, TimeUnit.SECONDS)); + Assertions.assertEquals(exact, cache.get(key, fallback, () -> Assertions.fail()).orElseThrow()); + Assertions.assertEquals(2, cache.cachedFileCount(key).orElseThrow()); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + @Test + void disabledAndWeightRejectedCachesDoNotStartUnretainedLoads() { + for (Map properties : List.of( + Map.of("meta.cache.deltalake.statistics.enable", "false"), + Map.of("meta.cache.deltalake.statistics.ttl-second", "0"), + Map.of("meta.cache.deltalake.statistics.max-weight", "1"))) { + QueueExecutor executor = new QueueExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, properties, executor); + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> Assertions.fail())); + Assertions.assertTrue(executor.tasks.isEmpty()); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + cache.fileCountObserver(key, () -> collected).accept(2); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + Assertions.assertTrue(executor.tasks.isEmpty()); + } + } + } + + @Test + void queueRejectionAndLoadFailureRemainRetryable() { + QueueExecutor executor = new QueueExecutor(); + AtomicInteger submissions = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), task -> { + if (submissions.getAndIncrement() == 0) { + throw new RejectedExecutionException("full"); + } + executor.execute(task); + }); + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> collected)); + cache.get(key, fallback, () -> { + throw new IOException("unavailable log"); + }); + executor.runNext(); + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> Assertions.fail())); + cache.get(key, fallback, () -> collected); + executor.runNext(); + Assertions.assertEquals(exact, cache.get(key, fallback, () -> Assertions.fail()).orElseThrow()); + } + } + + @Test + void fileCountLookupDoesNotReloadFailuresOrRequireRowStatistics() { + QueueExecutor executor = new QueueExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + cache.get(key, fallback, () -> { + throw new IOException("unavailable log"); + }); + executor.runNext(); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + Assertions.assertTrue(executor.tasks.isEmpty()); + cache.get(key, fallback, () -> new Statistics(new ConnectorTableStatistics(-1, 300), 2)); + executor.runNext(); + Assertions.assertEquals(2, cache.cachedFileCount(key).orElseThrow()); + } + } + + @Test + void observedCountsDoNotStartStatisticsOrSurviveInvalidationThroughALateReport() { + QueueExecutor executor = new QueueExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + DeltaLakeStatisticsCache cache = new DeltaLakeStatisticsCache(owner, Map.of(), executor); + LongConsumer first = cache.fileCountObserver(key, () -> collected); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + first.accept(2); + Assertions.assertEquals(2, cache.cachedFileCount(key).orElseThrow()); + Assertions.assertTrue(executor.tasks.isEmpty()); + Assertions.assertTrue(cache.cachedFileCount(key("bob", "table-id", 0)).isEmpty()); + Assertions.assertTrue(cache.cachedFileCount(key("alice", "table-id", 1)).isEmpty()); + Assertions.assertTrue(cache.cachedFileCount(key("alice", "replacement-id", 0)).isEmpty()); + Assertions.assertTrue(cache.cachedFileCount( + new SnapshotKey(new TableKey("alice", "db", "table", "/moved"), "table-id", 0)).isEmpty()); + owner.invalidateTable("db", "table"); + first.accept(999); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + LongConsumer replacement = cache.fileCountObserver(key, () -> collected); + replacement.accept(2); + first.accept(999); + Assertions.assertEquals(2, cache.cachedFileCount(key).orElseThrow()); + // A later statistics request can start the same entry and still obtain its row/size result. + Assertions.assertEquals(fallback, cache.get(key, fallback, () -> Assertions.fail("entry replaced"))); + executor.runNext(); + Assertions.assertEquals(Optional.of(exact), cache.get(key, fallback, () -> Assertions.fail())); + owner.invalidateCatalog(); + replacement.accept(999); + Assertions.assertTrue(cache.cachedFileCount(key).isEmpty()); + } + } + + @Test + void scanSchedulingUsesTheCurrentCrcWithoutCollectingStatistics() throws Exception { + Path table = createTable(); + add(table, "a.parquet", 100, 10L); + add(table, "b.parquet", 200, 20L); + checksum(table, 0, 2, 300); + try (Connector connector = new DeltaLakeConnectorProvider().create(Map.of("warehouse", warehouse.toString()), + new RecordingConnectorContext()); ConnectorMetadata metadata = connector.getMetadata(null)) { + TestingConnectorSession session = new TestingConnectorSession("crc"); + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "table").orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(session, handle, Optional.empty(), false)); + Assertions.assertEquals(2, provider.streamingSplitEstimate(new TestingConnectorSession("boundary", + Map.of("num_files_in_batch_mode", "2")), handle, Optional.empty(), false)); + } + } + + @Test + void countsActiveFilesAtEachVersionAndUsesOnlyMatchingCrc() throws Exception { + Path table = createTable(); + add(table, "a.parquet", 100, 10L); + add(table, "b.parquet", 200, 20L); + checksum(table, 0, 2, 300); + Snapshot first = snapshot(table, 0); + Assertions.assertEquals(collected, DeltaLakeStatisticsCache.collect(first, engine)); + Assertions.assertEquals(fallback, DeltaKernelCompat.tableStatistics(first)); + Files.writeString(table.resolve("_delta_log/00000000000000000001.json"), + "{\"remove\":{\"path\":\"a.parquet\",\"deletionTimestamp\":1,\"dataChange\":true}}\n"); + Snapshot second = snapshot(table, 1); + Assertions.assertEquals(new Statistics(new ConnectorTableStatistics(20, 200), 1), + DeltaLakeStatisticsCache.collect(second, engine)); + Assertions.assertTrue(DeltaKernelCompat.tableStatistics(second).isEmpty(), "old CRC is not current statistics"); + } + + @Test + void emptyMissingAndInvalidStatisticsStayDistinct() throws Exception { + Path table = createTable(); + checksum(table, 0, 0, 0); + Assertions.assertEquals(new ConnectorTableStatistics(0, 0), + DeltaKernelCompat.tableStatistics(snapshot(table, 0)).orElseThrow()); + Assertions.assertEquals(new Statistics(new ConnectorTableStatistics(0, 0), 0), + DeltaLakeStatisticsCache.collect(snapshot(table, 0), engine)); + Files.delete(table.resolve("_delta_log/00000000000000000000.crc")); + add(table, "known.parquet", 100, 10L); + add(table, "unknown.parquet", 200, null); + Assertions.assertEquals(new Statistics(new ConnectorTableStatistics(-1, 300), 2), + DeltaLakeStatisticsCache.collect(snapshot(table, 0), engine)); + DeltaLakeTestTables.commit(table, 0, "table-id", new StructType().add("id", LongType.LONG)); + add(table, "invalid.parquet", 100, -1L); + Assertions.assertThrows(DorisConnectorException.class, + () -> DeltaLakeStatisticsCache.collect(snapshot(table, 0), engine)); + } + + @Test + void subtractsDeletionVectorsAndReplaysBothCheckpointFormats() throws Exception { + for (String fixture : List.of("dv-partitioned-with-checkpoint", "v2-checkpoint-json", "v2-checkpoint-parquet")) { + Path table = Path.of(getClass().getResource("/deltalake/" + fixture).toURI()); + Snapshot snapshot = TableManager.loadSnapshot(table.toString()).build(engine); + ConnectorTableStatistics result = DeltaLakeStatisticsCache.collect(snapshot, engine).tableStatistics(); + Assertions.assertEquals(fixture.startsWith("dv-") ? 35 : 10, result.getRowCount()); + Assertions.assertTrue(result.getDataSize() > 0); + } + } + + @Test + void metadataReportsCachedCountsAtThePinnedVersionAfterAnAppend() throws Exception { + Path table = createTable(); + add(table, "a.parquet", 100, 10L); + AtomicInteger backgroundAuthentication = new AtomicInteger(); + RecordingConnectorContext context = new RecordingConnectorContext() { + @Override + public T executeAuthenticated(Callable action) throws Exception { + if (Thread.currentThread().getName().equals("deltalake-statistics")) { + backgroundAuthentication.incrementAndGet(); + Assertions.assertSame(DeltaLakeRuntime.class.getClassLoader(), + Thread.currentThread().getContextClassLoader()); + } + return super.executeAuthenticated(action); + } + }; + try (Connector connector = new DeltaLakeConnectorProvider().create(Map.of("warehouse", warehouse.toString()), + context); ConnectorMetadata metadata = connector.getMetadata(null)) { + TestingConnectorSession first = new TestingConnectorSession("first"); + ConnectorTableHandle handle = metadata.getTableHandle(first, "db", "table").orElseThrow(); + ConnectorMvccSnapshot pin = metadata.beginQuerySnapshot(first, handle).orElseThrow(); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + Assertions.assertTrue(provider.streamingSplitEstimate(first, handle, Optional.empty(), false) >= 0); + Assertions.assertEquals(10, await(() -> metadata.getTableStatistics(first, handle), 10).getRowCount()); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(first, handle, Optional.empty(), false)); + TestingConnectorSession boundary = new TestingConnectorSession("boundary", + Map.of("num_files_in_batch_mode", "1")); + Assertions.assertEquals(1, provider.streamingSplitEstimate(boundary, handle, Optional.empty(), false)); + Assertions.assertTrue(backgroundAuthentication.get() > 0); + Files.writeString(table.resolve("_delta_log/00000000000000000001.json"), + "{\"remove\":{\"path\":\"a.parquet\",\"deletionTimestamp\":1,\"dataChange\":true}}\n"); + connector.invalidateTable("db", "table"); + TestingConnectorSession next = new TestingConnectorSession("next"); + ConnectorTableHandle latest = metadata.getTableHandle(next, "db", "table").orElseThrow(); + Assertions.assertTrue(provider.streamingSplitEstimate(next, latest, Optional.empty(), false) >= 0); + Assertions.assertEquals(0, await(() -> metadata.getTableStatistics(next, latest), 0).getRowCount()); + Assertions.assertEquals(-1, provider.streamingSplitEstimate(next, latest, Optional.empty(), false)); + Assertions.assertEquals(10, await(() -> metadata.getTableStatistics(next, latest, pin), 10).getRowCount()); + } + } + + private static SnapshotKey key(String user, String id, long version) { + return new SnapshotKey(new TableKey(user, "db", "table", "/table"), id, version); + } + + private Path createTable() throws IOException { + Path table = warehouse.resolve("db/table"); + DeltaLakeTestTables.commit(table, 0, "table-id", new StructType().add("id", LongType.LONG)); + return table; + } + + private Snapshot snapshot(Path table, long version) { + return TableManager.loadSnapshot(table.toString()).atVersion(version).build(engine); + } + + private void add(Path table, String path, long size, Long records) throws IOException { + String stats = records == null ? "" : ",\"stats\":\"{\\\"numRecords\\\":" + records + "}\""; + Files.writeString(table.resolve("_delta_log/00000000000000000000.json"), + "{\"add\":{\"path\":\"" + path + "\",\"partitionValues\":{},\"size\":" + size + + ",\"modificationTime\":0,\"dataChange\":true" + stats + "}}\n", StandardOpenOption.APPEND); + } + + private void checksum(Path table, long version, long files, long bytes) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + List actions = Files.readAllLines(table.resolve("_delta_log/00000000000000000000.json")); + Files.writeString(table.resolve(String.format("_delta_log/%020d.crc", version)), mapper.writeValueAsString(Map.of( + "tableSizeBytes", bytes, "numFiles", files, "numMetadata", 1, "numProtocol", 1, + "metadata", mapper.readTree(actions.get(1)).get("metaData"), + "protocol", mapper.readTree(actions.get(0)).get("protocol")))); + } + + private static ConnectorTableStatistics await(Callable> action, long rows) + throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + do { + Optional result = action.call(); + if (result.isPresent() && result.get().getRowCount() == rows) { + return result.get(); + } + Thread.sleep(10); + } while (System.nanoTime() < deadline); + return Assertions.fail("statistics did not become ready for row count " + rows); + } + + private static final class QueueExecutor implements Executor { + private final Queue tasks = new ConcurrentLinkedQueue<>(); + + @Override + public void execute(Runnable task) { + tasks.add(task); + } + + void runNext() { + tasks.remove().run(); + } + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTestTables.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTestTables.java new file mode 100644 index 00000000000000..1f7ce2fd087384 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTestTables.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.kernel.types.StructType; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +final class DeltaLakeTestTables { + private DeltaLakeTestTables() { + } + + static void commit(Path table, long version, String id, StructType schema) throws IOException { + commit(table, version, id, schema, Collections.emptyList()); + } + + static void commit(Path table, long version, String id, StructType schema, List partitions) + throws IOException { + commit(table, version, id, schema, partitions, Map.of("minReaderVersion", 1, "minWriterVersion", 2)); + } + + static void commit(Path table, long version, String id, StructType schema, List partitions, + Map protocolFields) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + Path log = Files.createDirectories(table.resolve("_delta_log")); + String protocol = mapper.writeValueAsString(Map.of("protocol", protocolFields)); + String metadata = mapper.writeValueAsString(Map.of("metaData", Map.of( + "id", id, "format", Map.of("provider", "parquet", "options", Collections.emptyMap()), + "schemaString", schema.toJson(), "partitionColumns", partitions, + "configuration", Collections.emptyMap(), "createdTime", 0))); + Files.writeString(log.resolve(String.format("%020d.json", version)), protocol + "\n" + metadata + "\n"); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMappingTest.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMappingTest.java new file mode 100644 index 00000000000000..e6ef2d3a6a1f21 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/DeltaLakeTypeMappingTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.DorisConnectorException; + +import io.delta.kernel.types.ArrayType; +import io.delta.kernel.types.BinaryType; +import io.delta.kernel.types.BooleanType; +import io.delta.kernel.types.ByteType; +import io.delta.kernel.types.DataType; +import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; +import io.delta.kernel.types.DoubleType; +import io.delta.kernel.types.FloatType; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; +import io.delta.kernel.types.ShortType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; +import io.delta.kernel.types.TimestampType; +import io.delta.kernel.types.VariantType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +class DeltaLakeTypeMappingTest { + @Test + void mapsPrimitiveTypes() { + Map types = Map.of( + BooleanType.BOOLEAN, "BOOLEAN", ByteType.BYTE, "TINYINT", + ShortType.SHORT, "SMALLINT", IntegerType.INTEGER, "INT", LongType.LONG, "BIGINT", + FloatType.FLOAT, "FLOAT", DoubleType.DOUBLE, "DOUBLE", DateType.DATE, "DATEV2", + StringType.STRING, "STRING", BinaryType.BINARY, "STRING"); + types.forEach((source, target) -> Assertions.assertEquals(ConnectorType.of(target), map(source))); + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 38, 18), map(new DecimalType(38, 18))); + Assertions.assertEquals(ConnectorType.of("VARBINARY"), + DeltaLakeTypeMapping.toConnectorType(BinaryType.BINARY, false, true)); + } + + @Test + void distinguishesInstantAndWallClockTimestamps() { + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), map(TimestampType.TIMESTAMP)); + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 6, 0), + DeltaLakeTypeMapping.toConnectorType(TimestampType.TIMESTAMP, true, false)); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), + DeltaLakeTypeMapping.toConnectorType(TimestampNTZType.TIMESTAMP_NTZ, true, false)); + } + + @Test + void lowercasesNestedNamesAndPreservesNullability() { + StructType nested = new StructType().add("MiXeD", IntegerType.INTEGER, false); + ConnectorType array = map(new ArrayType(nested, false)); + Assertions.assertFalse(array.isChildNullable(0)); + Assertions.assertEquals("mixed", array.getChildren().get(0).getFieldNames().get(0)); + Assertions.assertFalse(array.getChildren().get(0).isChildNullable(0)); + ConnectorType map = map(new MapType(StringType.STRING, new ArrayType(nested, true), false)); + Assertions.assertFalse(map.isChildNullable(1)); + Assertions.assertEquals("mixed", + map.getChildren().get(1).getChildren().get(0).getFieldNames().get(0)); + } + + @Test + void rejectsTypesOutsideM1() { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> map(VariantType.VARIANT)); + Assertions.assertTrue(error.getMessage().contains("variant")); + Assertions.assertThrows(DorisConnectorException.class, () -> map(new StringType("SPARK.UTF8_LCASE"))); + } + + private static ConnectorType map(DataType type) { + return DeltaLakeTypeMapping.toConnectorType(type, false, false); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/RecordingConnectorContext.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/RecordingConnectorContext.java new file mode 100644 index 00000000000000..9e7829b57fa506 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/RecordingConnectorContext.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorContext; + +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; + +class RecordingConnectorContext implements ConnectorContext { + final AtomicInteger authenticationCalls = new AtomicInteger(); + volatile ClassLoader observedClassLoader; + + @Override + public String getCatalogName() { + return "delta_test"; + } + + @Override + public long getCatalogId() { + return 42; + } + + @Override + public T executeAuthenticated(Callable action) throws Exception { + authenticationCalls.incrementAndGet(); + observedClassLoader = Thread.currentThread().getContextClassLoader(); + return action.call(); + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/TestingConnectorSession.java b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/TestingConnectorSession.java new file mode 100644 index 00000000000000..e8e891e92fa828 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/java/org/apache/doris/connector/deltalake/TestingConnectorSession.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.deltalake; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +class TestingConnectorSession implements ConnectorSession { + private final String queryId; + private final Map properties; + private final ConnectorStatementScope scope = new ConnectorStatementScope() { + private volatile boolean closed; + private final Map values = new ConcurrentHashMap<>(); + + private final Map planning = new ConcurrentHashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + checkOpen(); + return (T) values.computeIfAbsent(key, ignored -> loader.get()); + } + + @Override + @SuppressWarnings("unchecked") + public T getIfPresent(String key) { + return closed ? null : (T) values.get(key); + } + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsentForPlanning(String key, Supplier loader) { + checkOpen(); + return (T) planning.computeIfAbsent(key, ignored -> loader.get()); + } + + @Override + public void endPlanning() { + planning.clear(); + } + + @Override + public void closeAll() { + closed = true; + endPlanning(); + values.clear(); + } + + private void checkOpen() { + if (closed) { + throw new CancellationException("Statement scope is closed"); + } + } + }; + + TestingConnectorSession(String queryId) { + this(queryId, Collections.emptyMap()); + } + + TestingConnectorSession(String queryId, Map properties) { + this.queryId = queryId; + this.properties = Map.copyOf(properties); + } + + @Override + public String getQueryId() { + return queryId; + } + + @Override + public String getUser() { + return "delta_user"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en-US"; + } + + @Override + public long getCatalogId() { + return 42; + } + + @Override + public String getCatalogName() { + return "delta_test"; + } + + @Override + public T getProperty(String name, Class type) { + return type.cast(properties.get(name)); + } + + @Override + public Map getSessionProperties() { + return properties; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } +} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/LICENSE.txt b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/LICENSE.txt new file mode 100644 index 00000000000000..a04059199fb38a --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/LICENSE.txt @@ -0,0 +1,388 @@ +Copyright (2021) The Delta Lake Project Authors. All rights reserved. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + + +------------------------------------------------------------------------- +This project includes code derived from the Apache Spark project. +The individual files containing this code carry the original Apache Spark +license, which is reproduced here as well: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/README.md b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/README.md new file mode 100644 index 00000000000000..0a6018354bcdcd --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/README.md @@ -0,0 +1,16 @@ +# Delta Kernel metadata fixtures + +Unmodified `_delta_log` files (including Parquet checkpoints and V2 sidecars) from +the Delta Lake project's Apache-2.0-licensed golden tables. Data files and Hadoop +`.crc` files are omitted: these tests exercise snapshot replay and scan planning. + +- Release: v4.4.1 +- Commit: `cebdc44dc07d331383e72811177af4ffc8638990` +- Source: `connectors/golden-tables/src/main/resources/golden` +- Tables: `dv-partitioned-with-checkpoint`, `v2-checkpoint-json`, `v2-checkpoint-parquet` +- Upstream: https://github.com/delta-io/delta/tree/cebdc44dc07d331383e72811177af4ffc8638990/connectors/golden-tables +- Source archive SHA-256: `a9b81e3e0a10c4b32afec823b5a2803eab57c06f542664478ecd438b2b982339` + +Expected latest snapshots: version 15 with 15 files, 37 physical records and two +deleted positions for the classic checkpoint table; version 2 with four files +and ten records for each V2 checkpoint table. diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000000.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..056e0b9d346318 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000000.json @@ -0,0 +1,23 @@ +{"commitInfo":{"timestamp":1688691721441,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"20","numOutputRows":"50","numOutputBytes":"14678"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"bc4bfee2-8faf-4995-bfa1-a3c4930cbf22"}} +{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1688691715853}} +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} +{"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=1/part-00000-a1586fa1-50e8-4f06-858a-b43b2e83010b.c000.snappy.parquet","partitionValues":{"part":"1"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":1,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":21,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=3/part-00000-319bea86-657f-4431-9b26-949dba99cf2c.c000.snappy.parquet","partitionValues":{"part":"3"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":3,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":23,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=4/part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet","partitionValues":{"part":"4"},"size":735,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":4,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":24,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=5/part-00000-5c963f16-d5b8-4f8b-8d8a-0e3403228be2.c000.snappy.parquet","partitionValues":{"part":"5"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":5,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":15,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=6/part-00000-be524334-115d-4d01-8614-e1bc8c630926.c000.snappy.parquet","partitionValues":{"part":"6"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":6,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":16,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=7/part-00000-33cc19fc-3607-4ea7-ab6d-af4e3ebf62c4.c000.snappy.parquet","partitionValues":{"part":"7"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":7,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":17,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=8/part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet","partitionValues":{"part":"8"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":8,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":18,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=9/part-00000-e4012c8c-cc60-44c0-babb-8c5d264a3a31.c000.snappy.parquet","partitionValues":{"part":"9"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":9,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":19,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=0/part-00001-24cdbe06-d3dc-449f-bd38-575228ca42a7.c000.snappy.parquet","partitionValues":{"part":"0"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":30,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":40,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=1/part-00001-d7e5d32a-55fa-410a-afee-adcdf46bc859.c000.snappy.parquet","partitionValues":{"part":"1"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":31,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":41,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=2/part-00001-ab1247be-1f77-41e6-a392-50a99b2db864.c000.snappy.parquet","partitionValues":{"part":"2"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":32,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":42,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=3/part-00001-afeef1dd-2517-49b9-873e-e9e6e8a74b19.c000.snappy.parquet","partitionValues":{"part":"3"},"size":731,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":33,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":43,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=4/part-00001-e63d3db6-9e97-4472-aacc-6af9fa44e73d.c000.snappy.parquet","partitionValues":{"part":"4"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":34,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":44,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=5/part-00001-f344b457-fbd0-4bc4-9502-2c07025e5bb1.c000.snappy.parquet","partitionValues":{"part":"5"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":25,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":45,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","partitionValues":{"part":"6"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":26,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":46,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=7/part-00001-986abb06-e672-4134-83d4-261752b236b8.c000.snappy.parquet","partitionValues":{"part":"7"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":27,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":47,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","partitionValues":{"part":"8"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":28,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":48,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} +{"add":{"path":"part=9/part-00001-c0430af8-a8e0-4b23-8776-b2fc549b3e4e.c000.snappy.parquet","partitionValues":{"part":"9"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":29,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":49,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000001.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..7b4eda2d684300 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691744588,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#463 = 0)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"13565","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"33e9e33f-49f5-421c-bf62-c3e71c55cd32"}} +{"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"0X9F0q2<2yJ-f)Gm2!e0","offset":1,"sizeInBytes":34,"cardinality":1}}} +{"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691741121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000002.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..685f921fe89b7e --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691752166,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#1939 = 2)\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"5256","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a9487f5e-84c9-44d8-a6d6-f7444e8405a9"}} +{"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"]m5]-UtmB0Rl$j","offset":1,"sizeInBytes":34,"cardinality":1}}} +{"remove":{"path":"part=8/part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet","deletionTimestamp":1688691765550,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"8"},"size":732}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000006.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000006.json new file mode 100644 index 00000000000000..3b628111329cb1 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000006.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691770236,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#7069 = 10)\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3110","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"7b578eea-e45c-4308-8c86-3c4c4f2bb9f9"}} +{"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Zcq?bs*HKQWCB[Sjf[.o","offset":1,"sizeInBytes":36,"cardinality":2}}} +{"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691769658,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"0X9F0q2<2yJ-f)Gm2!e0","offset":1,"sizeInBytes":34,"cardinality":1}}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000007.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000007.json new file mode 100644 index 00000000000000..cb693339679c11 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000007.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691774168,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#8351 = 12)\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2645","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"970ae4f8-d941-4029-b282-20e244e00cd1"}} +{"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"NZQXpd#3xxK!oBujr/=:","offset":1,"sizeInBytes":36,"cardinality":2}}} +{"remove":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","deletionTimestamp":1688691773564,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"2"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"]m5]-UtmB0Rl$j","offset":1,"sizeInBytes":34,"cardinality":1}}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000011.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000011.json new file mode 100644 index 00000000000000..11f254d86a802c --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000011.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1688691788332,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#13579 = 20)\"]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2084","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ad17e138-a1c2-4b6a-a373-e334c138c9e0"}} +{"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691788291,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"Zcq?bs*HKQWCB[Sjf[.o","offset":1,"sizeInBytes":36,"cardinality":2}}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000012.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000012.json new file mode 100644 index 00000000000000..696d56251f1be8 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000012.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1688691791603,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#15220 = 22)\"]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1870","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"68c7ac7c-6e73-4aa4-864f-ef374e110de6"}} +{"remove":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","deletionTimestamp":1688691791568,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"2"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"NZQXpd#3xxK!oBujr/=:","offset":1,"sizeInBytes":36,"cardinality":2}}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000013.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000013.json new file mode 100644 index 00000000000000..53759c69718390 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000013.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1688691796131,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#16548 = 24)\"]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3461","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"4e6f5590-7630-44d1-bc1c-5a2bfd637c4f"}} +{"remove":{"path":"part=4/part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet","deletionTimestamp":1688691796068,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"4"},"size":735,"deletionVector":{"storageType":"u","pathOrInlineDv":"+V9Oq([R6mTaoM.}Isa)","offset":1,"sizeInBytes":36,"cardinality":2}}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000014.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000014.json new file mode 100644 index 00000000000000..e9eb0375f6b55e --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000014.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691798638,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#17876 = 26)\"]"},"readVersion":13,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1451","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"34f3a5ca-2166-4b49-a92b-f6b70ce4cd89"}} +{"add":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","partitionValues":{"part":"6"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":26,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":46,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"]g#Gi8g29gLy&KMkGJr?","offset":1,"sizeInBytes":34,"cardinality":1}}} +{"remove":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","deletionTimestamp":1688691798060,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"6"},"size":736}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000015.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000015.json new file mode 100644 index 00000000000000..c4d415d1903933 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/00000000000000000015.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1688691801431,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#19379 = 28)\"]"},"readVersion":14,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1804","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ace020dc-2202-4814-bbd4-36fc7a2a7988"}} +{"add":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","partitionValues":{"part":"8"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":28,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":48,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"7NnC4LX-RqU.S:B4VD9n","offset":1,"sizeInBytes":34,"cardinality":1}}} +{"remove":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","deletionTimestamp":1688691800672,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"8"},"size":736}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/_last_checkpoint b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/_last_checkpoint new file mode 100644 index 00000000000000..93a5d0b0cc45da --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/dv-partitioned-with-checkpoint/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":10,"size":30,"sizeInBytes":20573,"numOfAddFiles":18,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues_parsed","type":{"type":"struct","fields":[{"name":"part","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"822eda70bb966b38d646f351ea753463"} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000000.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..daf36e1a7f1c7c --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1714496114594,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"f6282e54-afc6-4669-939b-0f8ba73062a0"}} +{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496114564}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000001.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..e057138c054279 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1714496114748,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"fddb3112-ca9b-48af-bf19-be23f1c36c22"}} +{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json new file mode 100644 index 00000000000000..d0831860a104f4 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json @@ -0,0 +1,5 @@ +{"checkpointMetadata":{"version":2}} +{"sidecar":{"path":"00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet","sizeInBytes":9367,"modificationTime":1714496115780}} +{"sidecar":{"path":"00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet","sizeInBytes":9296,"modificationTime":1714496115788}} +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} +{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..d1ca6a702d4619 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/00000000000000000002.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1714496115090,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"a76e8fca-8bab-42cc-9618-77f8c536968c"}} +{"add":{"path":"part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115046,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496115048,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_last_checkpoint b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_last_checkpoint new file mode 100644 index 00000000000000..e223a53503a236 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":2,"size":9,"sizeInBytes":19554,"numOfAddFiles":4,"v2Checkpoint":{"path":"00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json","sizeInBytes":891,"modificationTime":1714496115810,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}},{"checkpointMetadata":{"version":2}}],"sidecarFiles":[{"path":"00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet","sizeInBytes":9367,"modificationTime":1714496115780},{"path":"00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet","sizeInBytes":9296,"modificationTime":1714496115788}]},"checksum":"d09f95a326aab562c60d415a32ddd216"} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet new file mode 100644 index 00000000000000..bf03a73fdefb36 Binary files /dev/null and b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet differ diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet new file mode 100644 index 00000000000000..daaee88f6be88c Binary files /dev/null and b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-json/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet differ diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000000.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..b0b1c2a6488468 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1714496109365,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"7517176e-cff7-46ac-b133-3cf096e2620d"}} +{"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496109258}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000001.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..0d4da03985862d --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1714496110834,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"12ea26b9-c620-4104-95f6-654bcaabdda6"}} +{"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}} +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet new file mode 100644 index 00000000000000..d5d301d8d440de Binary files /dev/null and b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet differ diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.json b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..c424c601857a22 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/00000000000000000002.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1714496112086,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"c9f86c17-1b30-44e7-873d-1e2102f54b0f"}} +{"add":{"path":"part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496112068,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496112071,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_last_checkpoint b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_last_checkpoint new file mode 100644 index 00000000000000..c105e156cf4a54 --- /dev/null +++ b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":2,"size":9,"sizeInBytes":37269,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"checkpointMetadata","type":{"type":"struct","fields":[{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"sidecar","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"v2Checkpoint":{"path":"00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet","sizeInBytes":18634,"modificationTime":1714496114276,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}},{"checkpointMetadata":{"version":2}}],"sidecarFiles":[{"path":"00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet","sizeInBytes":9268,"modificationTime":1714496113961},{"path":"00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet","sizeInBytes":9367,"modificationTime":1714496113961}]},"checksum":"f81aaf268542b71bb3fc9b63f754f9df"} diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet new file mode 100644 index 00000000000000..dd7b75a24a75f5 Binary files /dev/null and b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet differ diff --git a/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet new file mode 100644 index 00000000000000..aeda4fde162596 Binary files /dev/null and b/fe/fe-connector/fe-connector-deltalake/src/test/resources/deltalake/v2-checkpoint-parquet/_delta_log/_sidecars/00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet differ diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/DeltaSiblingProperties.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/DeltaSiblingProperties.java new file mode 100644 index 00000000000000..7dec0997cdd1fa --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/DeltaSiblingProperties.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import java.util.HashMap; +import java.util.Map; + +/** The gateway and its Delta sibling share HMS/storage settings and catalog-owned caches. */ +final class DeltaSiblingProperties { + private DeltaSiblingProperties() { + } + + static Map synthesize(Map gatewayProperties) { + Map properties = new HashMap<>(gatewayProperties); + // The sibling must resolve this gateway's HMS tables, even if a filesystem flavor was supplied. + properties.put("deltalake.catalog.type", "hms"); + return properties; + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 4a8016822b11e1..be55a743569a95 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -82,6 +82,7 @@ public class HiveConnector implements Connector { // string HudiConnectorProvider registers, which declares isStandaloneCatalogType() == false so that the // engine can never build a hudi catalog. Sibling lookup does not consult that flag, so this stays valid. private static final String HUDI_CONNECTOR_TYPE = "hudi"; + private static final String DELTA_CONNECTOR_TYPE = "deltalake"; private final Map properties; // Bound + validated once here: an existing HiveConnector always has usable catalog properties. Built on @@ -129,6 +130,8 @@ public class HiveConnector implements Connector { // zip (a second AWS SDK would poison S3 JVM-wide). Held ONLY as the parent-first Connector interface and // NEVER cast (a cast would CCE across the loader split). private volatile Connector hudiSibling; + // Like the other siblings, Delta is created only for a matching HMS table in its own plugin loader. + private volatile Connector deltaSibling; // Set FIRST in close(), under the same monitor the getOrCreate* holders below take, and never cleared. // Every one of those holders is a plain double-checked lazy build over a field close() nulls out, so @@ -218,7 +221,8 @@ private static String rootCauseMessage(Throwable t) { */ HiveConnectorMetadata newMetadata(HmsClient client) { return new HiveConnectorMetadata(client, props, context, - this::getOrCreateIcebergSibling, this::getOrCreateHudiSibling, this::resolveSiblingOwnerLabeled, + this::getOrCreateIcebergSibling, this::getOrCreateHudiSibling, this::getOrCreateDeltaSibling, + this::resolveSiblingOwnerLabeled, fileListingCache, partitionViewCache); } @@ -247,6 +251,10 @@ SiblingOwner resolveSiblingOwnerLabeled(ConnectorTableHandle handle) { if (hudi != null && hudi.ownsHandle(handle)) { return new SiblingOwner(hudi, SiblingOwner.HUDI_LABEL); } + Connector delta = deltaSibling; + if (delta != null && delta.ownsHandle(handle)) { + return new SiblingOwner(delta, SiblingOwner.DELTA_LABEL); + } throw new DorisConnectorException("Cannot route a foreign table handle in catalog '" + context.getCatalogName() + "': no embedded sibling connector owns it"); } @@ -472,6 +480,10 @@ private void forEachBuiltSibling(Consumer action) { if (hudi != null) { action.accept(hudi); } + Connector delta = deltaSibling; + if (delta != null) { + action.accept(delta); + } } /** The connector-owned directory-listing cache, exposed for unit tests (mirrors iceberg manifestCacheForTest). */ @@ -588,6 +600,26 @@ Connector getOrCreateHudiSibling() { return hudiSibling; } + /** Builds the Delta sibling under the same close/build monitor as the other plugins. */ + Connector getOrCreateDeltaSibling() { + if (deltaSibling == null) { + synchronized (this) { + throwIfClosed(); + if (deltaSibling == null) { + Connector sibling = context.createSiblingConnector( + DELTA_CONNECTOR_TYPE, DeltaSiblingProperties.synthesize(properties)); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot serve delta-on-HMS tables in catalog '" + context.getCatalogName() + + "': the deltalake connector plugin is not available"); + } + deltaSibling = sibling; + } + } + } + return deltaSibling; + } + private HmsClient createClient() { // No removed-type / required-URI check here: both are invariants of HiveCatalogProperties, which this // connector's constructor built. A catalog created before the type was removed, or one carrying no @@ -771,6 +803,7 @@ public void close() throws IOException { HmsClient client; Connector iceberg; Connector hudi; + Connector delta; synchronized (this) { closed = true; client = hmsClient; @@ -779,11 +812,14 @@ public void close() throws IOException { icebergSibling = null; hudi = hudiSibling; hudiSibling = null; + delta = deltaSibling; + deltaSibling = null; } IOException failure = closeStage(metaCache, "metadata cache", null); failure = closeStage(client, "hive metastore client", failure); failure = closeStage(iceberg, "iceberg-on-HMS sibling connector", failure); failure = closeStage(hudi, "hudi-on-HMS sibling connector", failure); + failure = closeStage(delta, "delta-on-HMS sibling connector", failure); if (failure != null) { throw failure; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 26ed549b41ceca..d2b1ad9b86dd67 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -279,6 +279,7 @@ public class HiveConnectorMetadata implements ConnectorMetadata { // HiveConnector.getOrCreateHudiSibling; used ONLY through the parent-first Connector / ConnectorMetadata // interfaces — its concrete hudi types are never cast here (cross-loader CCE). private final Supplier hudiSiblingSupplier; + private final Supplier deltaSiblingSupplier; // Resolves the embedded sibling connector that OWNS a foreign (non-hive) table handle, for the per-handle // guard-and-forward methods below. Backed by HiveConnector.resolveSiblingOwner (a 3-way ownsHandle dispatch // over the ALREADY-BUILT iceberg / hudi siblings). Used ONLY through the parent-first Connector / @@ -324,7 +325,7 @@ public HiveConnectorMetadata(HmsClient hmsClient, HiveCatalogProperties properti } /** - * Full ctor used by {@link HiveConnector#newMetadata}, adding the PERF-06 derived partition-view cache + * Constructor for callers without a Delta sibling, adding the PERF-06 derived partition-view cache * (cache A): {@code partitionViewCache} memoizes {@link #listPartitions}'s built * {@code List}, keyed by {@code (db, table, -1, -1)}. {@code null} for the * convenience/test ctors (no cross-query derived layer -> compute directly every call). @@ -335,10 +336,23 @@ public HiveConnectorMetadata(HmsClient hmsClient, HiveCatalogProperties properti Function siblingOwnerResolver, HiveFileListingCache fileListingCache, ConnectorMetadataCache> partitionViewCache) { + this(hmsClient, properties, context, icebergSiblingSupplier, hudiSiblingSupplier, () -> { + throw new DorisConnectorException("no deltalake sibling connector configured for this hive metadata"); + }, siblingOwnerResolver, fileListingCache, partitionViewCache); + } + + /** Gateway constructor: all sibling factories are lazy and share the same owner resolver. */ + HiveConnectorMetadata(HmsClient hmsClient, HiveCatalogProperties properties, ConnectorContext context, + Supplier icebergSiblingSupplier, Supplier hudiSiblingSupplier, + Supplier deltaSiblingSupplier, + Function siblingOwnerResolver, + HiveFileListingCache fileListingCache, + ConnectorMetadataCache> partitionViewCache) { this.hmsClient = hmsClient; this.context = context; this.icebergSiblingSupplier = icebergSiblingSupplier; this.hudiSiblingSupplier = hudiSiblingSupplier; + this.deltaSiblingSupplier = deltaSiblingSupplier; this.siblingOwnerResolver = siblingOwnerResolver; this.fileListingCache = fileListingCache; this.partitionViewCache = partitionViewCache; @@ -484,6 +498,10 @@ public Optional getTableHandle( if (tableType == HiveTableType.HUDI) { return hudiSiblingMetadata(session).getTableHandle(session, dbName, tableName); } + if (tableType == HiveTableType.DELTA) { + return memoizedSiblingMetadata(session, deltaSiblingSupplier.get(), SiblingOwner.DELTA_LABEL) + .getTableHandle(session, dbName, tableName); + } // Fail-loud parity with legacy HMSExternalTable.supportedHiveTable(), which threw on a null or // unrecognized input format instead of silently degrading (the old detector returned UNKNOWN). A view // short-circuits: legacy returns true for a view before the format check — a view has no data files so diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableFormatDetector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableFormatDetector.java index 45985e0a4a968f..7607d959483fbe 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableFormatDetector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableFormatDetector.java @@ -28,7 +28,7 @@ /** * Detects the actual table format of a table registered in HMS. * - *

HMS can store Hive, Hudi, and Iceberg tables under the same metastore. + *

HMS can store Hive, Hudi, Iceberg, and Delta Lake tables under the same metastore. * This class mirrors the detection logic from fe-core's * {@code HMSExternalTable.supportedIcebergTable/supportedHoodieTable/supportedHiveTable} * using only SPI types.

@@ -78,11 +78,12 @@ private HiveTableFormatDetector() { /** * Detect the table format from HMS table metadata. * - *

Detection order (matches fe-core): + *

Detection order: *

    *
  1. Iceberg: parameters contain {@code table_type=ICEBERG}
  2. *
  3. Hudi: input format in SUPPORTED_HUDI_INPUT_FORMATS or * {@code flink.connector=hudi}
  4. + *
  5. Delta Lake: parameters contain {@code spark.sql.sources.provider=delta}
  6. *
  7. Hive: input format in SUPPORTED_HIVE_INPUT_FORMATS
  8. *
  9. UNKNOWN: none of the above
  10. *
@@ -107,7 +108,12 @@ public static HiveTableType detect(HmsTableInfo tableInfo) { return HiveTableType.HUDI; } - // 3. Hive detection + // Delta tables can carry a generic Hive input format: their provider remains authoritative. + if (params != null && "delta".equalsIgnoreCase(params.get("spark.sql.sources.provider"))) { + return HiveTableType.DELTA; + } + + // Hive detection if (inputFormat != null && SUPPORTED_HIVE_INPUT_FORMATS.contains(inputFormat)) { return HiveTableType.HIVE; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableType.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableType.java index b7a8c31b000ff0..526d4e5b701c40 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableType.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveTableType.java @@ -21,9 +21,8 @@ * The detected format type of a Hive-registered table. * *

Tables registered in HMS may actually be backed by different engines. - * This enum distinguishes them. In Phase 1, the Hive connector only - * handles {@link #HIVE} tables; other types will be handled by their - * respective connector plugins in the future.

+ * This enum distinguishes them. The Hive connector serves {@link #HIVE} + * tables directly and delegates the other supported formats to their sibling plugins.

*/ public enum HiveTableType { @@ -36,6 +35,9 @@ public enum HiveTableType { /** Iceberg table registered in HMS (detected by table_type=ICEBERG parameter) */ ICEBERG, + /** Delta Lake table registered in HMS (spark.sql.sources.provider=delta). */ + DELTA, + /** Cannot determine the table format */ UNKNOWN; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/SiblingOwner.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/SiblingOwner.java index 9ef9c14215c20a..d9642c73ebca3d 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/SiblingOwner.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/SiblingOwner.java @@ -42,6 +42,8 @@ final class SiblingOwner { /** Owner label for the embedded hudi sibling — the funnel-key discriminator, shared by both routes. */ static final String HUDI_LABEL = "hudi"; + static final String DELTA_LABEL = "deltalake"; + private final Connector connector; private final String label; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorDeltaSiblingTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorDeltaSiblingTest.java new file mode 100644 index 00000000000000..eb625d7e3121df --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorDeltaSiblingTest.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +class HiveConnectorDeltaSiblingTest { + private static final String PARQUET = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"; + + @Test + void detectsDeltaBeforeGenericHiveFormatsAndPreservesOtherFormats() { + for (String provider : List.of("delta", "DELTA", "DeLtA")) { + Assertions.assertEquals(HiveTableType.DELTA, HiveTableFormatDetector.detect( + table(Map.of("spark.sql.sources.provider", provider)))); + } + Assertions.assertEquals(HiveTableType.HIVE, HiveTableFormatDetector.detect(table(Map.of()))); + Assertions.assertEquals(HiveTableType.ICEBERG, HiveTableFormatDetector.detect( + table(Map.of("table_type", "ICEBERG", "spark.sql.sources.provider", "delta")))); + Assertions.assertEquals(HiveTableType.HUDI, HiveTableFormatDetector.detect( + table(Map.of("flink.connector", "hudi", "spark.sql.sources.provider", "delta")))); + } + + @Test + void forcesHmsAndCopiesAllConnectionAndCacheProperties() { + Map input = new HashMap<>(Map.of( + "deltalake.catalog.type", "filesystem", "uri", "thrift://host:9083", + "s3.endpoint", "https://storage", "dfs.nameservices", "ns1", + "hive.metastore.client.principal", "hive/_HOST@REALM", + "meta.cache.deltalake.latest.ttl-second", "5")); + Map result = DeltaSiblingProperties.synthesize(input); + Assertions.assertEquals("hms", result.remove("deltalake.catalog.type")); + Assertions.assertEquals("filesystem", input.remove("deltalake.catalog.type")); + Assertions.assertEquals(input, result); + result.clear(); + Assertions.assertFalse(input.isEmpty()); + } + + @Test + void routesTheRawHandleSchemaAndScanThroughOneStatementMetadata() throws Exception { + DeltaContext context = new DeltaContext(); + try (HiveConnector gateway = gateway(context)) { + HiveConnectorMetadata metadata = gateway.newMetadata(hms(table(Map.of("spark.sql.sources.provider", "delta")))); + ConnectorSession session = new ScopeSession(0, "delta", new TestStatementScope()); + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "t").orElseThrow(); + Assertions.assertSame(context.delta.handle, handle); + Assertions.assertEquals(SiblingOwner.DELTA_LABEL, gateway.resolveSiblingOwnerLabeled(handle).label()); + Assertions.assertSame(context.delta.schema, metadata.getTableSchema(session, handle)); + Assertions.assertSame(context.delta.scans, gateway.getScanPlanProvider(handle)); + Assertions.assertEquals(1, context.delta.metadataCalls); + Assertions.assertEquals(1, context.builds); + } + } + + @Test + void plainHiveAndRefreshNeverBuildUnneededPlugins() throws Exception { + DeltaContext context = new DeltaContext(); + try (HiveConnector gateway = gateway(context)) { + HiveConnectorMetadata metadata = gateway.newMetadata(hms(table(Map.of()))); + Assertions.assertInstanceOf(HiveTableHandle.class, metadata.getTableHandle( + new ScopeSession(0, "hive", new TestStatementScope()), "db", "t").orElseThrow()); + gateway.invalidateTable("db", "t"); + gateway.invalidateDb("db"); + gateway.invalidateAll(); + Assertions.assertEquals(0, context.builds); + } + Assertions.assertEquals(0, context.delta.closes); + } + + @Test + void refreshAndCloseReachTheDeltaSiblingAndPreventRecreation() throws Exception { + DeltaContext context = new DeltaContext(); + HiveConnector gateway = gateway(context); + gateway.getOrCreateDeltaSibling(); + gateway.invalidateTable("db", "t"); + gateway.invalidateDb("db"); + gateway.invalidateAll(); + Assertions.assertEquals(List.of("db.t", "db", "all"), context.delta.invalidated); + gateway.close(); + gateway.close(); + Assertions.assertEquals(1, context.delta.closes); + Assertions.assertThrows(DorisConnectorException.class, gateway::getOrCreateDeltaSibling); + Assertions.assertEquals(1, context.builds); + } + + @Test + void unavailablePluginFailsClearlyAndCanRecover() throws Exception { + DeltaContext context = new DeltaContext(); + context.available = false; + try (HiveConnector gateway = gateway(context)) { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + gateway::getOrCreateDeltaSibling); + Assertions.assertTrue(error.getMessage().contains("deltalake connector plugin is not available")); + Assertions.assertTrue(error.getMessage().contains("test_catalog")); + context.available = true; + Assertions.assertSame(context.delta, gateway.getOrCreateDeltaSibling()); + Assertions.assertEquals(2, context.builds); + } + } + + private static HiveConnector gateway(DeltaContext context) { + return new HiveConnector(Map.of("hive.metastore.uris", "thrift://host:9083"), context); + } + + private static HmsTableInfo table(Map parameters) { + return HmsTableInfo.builder().dbName("db").tableName("t").tableType("EXTERNAL_TABLE") + .inputFormat(PARQUET).parameters(parameters).build(); + } + + private static HmsClient hms(HmsTableInfo table) { + return (HmsClient) Proxy.newProxyInstance(HmsClient.class.getClassLoader(), new Class[] {HmsClient.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "tableExists": + return true; + case "getTable": + return table; + default: + throw new AssertionError("Unexpected HMS call: " + method.getName()); + } + }); + } + + private static final class DeltaContext extends FakeConnectorContext { + private final DeltaSibling delta = new DeltaSibling(); + private boolean available = true; + private int builds; + + @Override + public Connector createSiblingConnector(String type, Map properties) { + Assertions.assertEquals("deltalake", type, "unrelated sibling must not be created"); + Assertions.assertEquals("hms", properties.get("deltalake.catalog.type")); + builds++; + return available ? delta : null; + } + } + + private static final class DeltaSibling implements Connector { + private final ConnectorTableHandle handle = new ConnectorTableHandle() {}; + private final ConnectorTableSchema schema = new ConnectorTableSchema("t", List.of(), "deltalake", Map.of()); + private final List invalidated = new ArrayList<>(); + private int metadataCalls; + private int closes; + private final ConnectorScanPlanProvider scans = new ConnectorScanPlanProvider() { + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + return List.of(); + } + }; + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + metadataCalls++; + return new ConnectorMetadata() { + @Override + public Optional getTableHandle(ConnectorSession session, String db, String table) { + return Optional.of(handle); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) { + Assertions.assertSame(handle, table); + return schema; + } + }; + } + + @Override + public boolean ownsHandle(ConnectorTableHandle table) { + return table == handle; + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle table) { + Assertions.assertSame(handle, table); + return scans; + } + + @Override + public void invalidateTable(String db, String table) { + invalidated.add(db + "." + table); + } + + @Override + public void invalidateDb(String db) { + invalidated.add(db); + } + + @Override + public void invalidateAll() { + invalidated.add("all"); + } + + @Override + public void close() { + closes++; + } + } +} diff --git a/fe/fe-connector/fe-connector-metastore-deltalake/pom.xml b/fe/fe-connector/fe-connector-metastore-deltalake/pom.xml new file mode 100644 index 00000000000000..47da869e68b2b4 --- /dev/null +++ b/fe/fe-connector/fe-connector-metastore-deltalake/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-metastore-deltalake + jar + Doris FE Connector Metastore Delta Lake + Delta Lake filesystem metastore properties and provider. + + + + ${project.groupId} + fe-connector-metastore-spi + ${project.version} + + + ${project.groupId} + fe-connector-metastore-api + ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + + org.junit.jupiter + junit-jupiter + test + + + + + doris-fe-connector-metastore-deltalake + + + org.apache.maven.plugins + maven-dependency-plugin + + + + copy-plugin-deps + none + + + + + + diff --git a/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProperties.java b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProperties.java new file mode 100644 index 00000000000000..abf580955855a9 --- /dev/null +++ b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProperties.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.metastore.deltalake.fs; + +import org.apache.doris.connector.metastore.FileSystemMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.AbstractMetaStoreProperties; +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; + +import java.util.Map; + +/** + * DeltaLake filesystem (warehouse-only) metastore backend facts. The storage config is overlaid by the + * connector at catalog-build time, so this impl carries only the warehouse and declares + * {@link #needsStorage()} true. + */ +public final class DeltaLakeFileSystemMetaStoreProperties extends AbstractMetaStoreProperties + implements FileSystemMetaStoreProperties { + + private DeltaLakeFileSystemMetaStoreProperties(Map raw) { + super(raw); + } + + public static DeltaLakeFileSystemMetaStoreProperties of(Map raw) { + DeltaLakeFileSystemMetaStoreProperties props = new DeltaLakeFileSystemMetaStoreProperties(raw); + ConnectorPropertiesUtils.bindConnectorProperties(props, raw); + return props; + } + + @Override + public String providerName() { + return "FILESYSTEM"; + } + + @Override + public boolean needsStorage() { + return true; + } + + @Override + public void validate() { + requireWarehouse(); + } + + @Override + public String getWarehouse() { + return warehouse; + } +} diff --git a/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProvider.java b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProvider.java new file mode 100644 index 00000000000000..cf90c8b6843b21 --- /dev/null +++ b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStoreProvider.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.metastore.deltalake.fs; + +import org.apache.doris.connector.metastore.FileSystemMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.MetaStoreProvider; + +import java.util.Map; + +/** Selects the Delta Lake filesystem backend through explicit metastore-type dispatch. */ +public final class DeltaLakeFileSystemMetaStoreProvider implements MetaStoreProvider { + + @Override + public boolean supportsType(String catalogType) { + return "filesystem".equalsIgnoreCase(catalogType); + } + + @Override + public FileSystemMetaStoreProperties bind(Map properties, + Map storageHadoopConfig) { + return DeltaLakeFileSystemMetaStoreProperties.of(properties); + } + + @Override + public String name() { + return "FILESYSTEM"; + } +} diff --git a/fe/fe-connector/fe-connector-metastore-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.metastore.spi.MetaStoreProvider b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.metastore.spi.MetaStoreProvider new file mode 100644 index 00000000000000..ed500dcf0f5e03 --- /dev/null +++ b/fe/fe-connector/fe-connector-metastore-deltalake/src/main/resources/META-INF/services/org.apache.doris.connector.metastore.spi.MetaStoreProvider @@ -0,0 +1 @@ +org.apache.doris.connector.metastore.deltalake.fs.DeltaLakeFileSystemMetaStoreProvider diff --git a/fe/fe-connector/fe-connector-metastore-deltalake/src/test/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStorePropertiesTest.java b/fe/fe-connector/fe-connector-metastore-deltalake/src/test/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStorePropertiesTest.java new file mode 100644 index 00000000000000..9ce6c2229c71ba --- /dev/null +++ b/fe/fe-connector/fe-connector-metastore-deltalake/src/test/java/org/apache/doris/connector/metastore/deltalake/fs/DeltaLakeFileSystemMetaStorePropertiesTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.connector.metastore.deltalake.fs; + +import org.apache.doris.connector.metastore.FileSystemMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.MetaStoreProviders; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +class DeltaLakeFileSystemMetaStorePropertiesTest { + @Test + void dispatchesFilesystemThroughServiceLoader() { + FileSystemMetaStoreProperties properties = (FileSystemMetaStoreProperties) MetaStoreProviders.bindForType( + "filesystem", Collections.singletonMap("warehouse", "s3://bucket/warehouse"), Collections.emptyMap()); + properties.validate(); + Assertions.assertEquals("s3://bucket/warehouse", properties.getWarehouse()); + Assertions.assertTrue(properties.needsStorage()); + } + + @Test + void requiresWarehouse() { + DeltaLakeFileSystemMetaStoreProperties properties = + DeltaLakeFileSystemMetaStoreProperties.of(Collections.emptyMap()); + Assertions.assertThrows(IllegalArgumentException.class, properties::validate); + } +} diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorPushdownOps.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorPushdownOps.java index add1b46ccdfc94..05240c7ba64ba0 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorPushdownOps.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorPushdownOps.java @@ -58,6 +58,11 @@ public interface ConnectorPushdownOps { * {@link #applyFilter} path never consults it: there the engine converts every conjunct and hands the * result to the connector regardless of what this method answers.

* + *

A per-table scan provider can implement this interface to further restrict the residuals it + * receives. Both the catalog metadata and that provider must accept CASTs. This lets a gateway + * honor the file reader of the selected table. Filter and projection application still use the + * catalog metadata.

+ * *

And on either path a CAST does not arrive as a CAST. The forward converter unwraps it and * pushes the child expression, so {@code CAST(dt AS INT) = 20240101} reaches the connector as * {@code dt = 20240101}, with nothing marking it as coerced — a connector cannot detect the situation by @@ -75,9 +80,9 @@ public interface ConnectorPushdownOps { *

Every shipped connector answers this deliberately; none of them merely inherits. jdbc, paimon, * maxcompute and the trino bridge return {@code false}; iceberg and elasticsearch state {@code true} at * their own metadata class, each recording what it does with the predicate and that {@code true} is an - * accepted risk rather than a safety claim. hive and hudi declare nothing because for them the switch is - * INERT — their scan planning ignores the residual filter entirely, and the predicate they do consume - * arrives on the {@link #applyFilter} path, which is never gated on this method. A new connector should + * accepted risk rather than a safety claim. Delta's scan provider returns {@code false} for both direct + * scans and Hive gateway scans. Native Hive and Hudi scan planning ignores the residual filter entirely; + * the predicates those readers consume arrive on the {@link #applyFilter} path. A new connector should * make the same deliberate choice, starting from {@code false}.

*/ default boolean supportsCastPredicatePushdown(ConnectorSession session) { diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index f6ff0f9aeda315..75ca4f0bc17c6c 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -66,6 +66,7 @@ under the License. fe-connector-metastore-paimon + fe-connector-metastore-deltalake fe-connector-metastore-iceberg fe-connector-metastore-hms fe-connector-es @@ -84,6 +85,7 @@ under the License. package); MUST build before fe-connector-paimon which depends on it. --> fe-connector-paimon-hive-shade fe-connector-paimon + fe-connector-deltalake fe-connector-hudi fe-connector-iceberg fe-connector-fluss diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 39f04ed38831cf..95790d799dd13d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -42,6 +42,7 @@ import org.apache.doris.connector.ConnectorStatementScopeImpl; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorPushdownOps; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorTaskScheduler; @@ -1894,7 +1895,7 @@ public List getSplits(int numBackends) throws UserException { List columns = buildColumnHandles(); tryPushDownProjection(columns); - Optional remainingFilter = buildRemainingFilter(); + Optional remainingFilter = buildRemainingFilter(scanProvider); // Pin the MVCC snapshot onto currentHandle AFTER projection/filter pushdown rebuilt it and // immediately before planScan consumes it, so the native split path reads at the pinned // snapshot. getSplits already declares UserException, so a getTargetTable() failure propagates. @@ -2199,7 +2200,7 @@ private boolean computeBatchMode() { boolean countPushdown = isTableLevelCountStarPushdown(); long estimate = readMetadata(() -> onPluginClassLoader(scanProvider, () -> scanProvider.streamingSplitEstimate( - connectorSession, currentHandle, buildRemainingFilter(), countPushdown))); + connectorSession, currentHandle, buildRemainingFilter(scanProvider), countPushdown))); if (estimate >= 0) { // Streaming does not know the final scanned-partition count before dispatch. Partition // rules need that count now; use synchronous planning, which publishes the connector count. @@ -2343,7 +2344,7 @@ private ConnectorScanRequest prepareBatchScan(QueryFinishCleanup queryFinishClea // request, just as getSplits does. Batch scans deliberately omit LIMIT and COUNT(*) pushdown. List columns = buildColumnHandles(); tryPushDownProjection(columns); - Optional remainingFilter = buildRemainingFilter(); + Optional remainingFilter = buildRemainingFilter(resolveScanProvider()); // The pre-pin streaming estimate is only a scheduling hint; the source reads the pinned handle. pinMvccSnapshot(); pinRewriteFileScope(); @@ -2748,7 +2749,7 @@ private ScanNodePropertiesResult getOrLoadPropertiesResult() { // (same unchecked error channel as create() above) rather than dropped. try { columns = buildColumnHandles(); - filter = buildRemainingFilter(); + filter = buildRemainingFilter(scanProvider); pinMvccSnapshot(); } catch (UserException e) { throw new RuntimeException("Failed to build column handles / pin MVCC snapshot" @@ -2961,7 +2962,7 @@ private ConnectorFilterConstraint buildFilterConstraint(List exprs) { * If no conjuncts remain, returns {@link Optional#empty()}. * Filters out CAST-containing predicates when the connector does not support CAST pushdown. */ - private Optional buildRemainingFilter() { + private Optional buildRemainingFilter(ConnectorScanPlanProvider scanProvider) { if (conjuncts == null || conjuncts.isEmpty()) { filteredToOriginalIndex = null; return Optional.empty(); @@ -2969,6 +2970,12 @@ private Optional buildRemainingFilter() { List pushableConjuncts = new ArrayList<>(); filteredToOriginalIndex = new ArrayList<>(); boolean supportsCast = metadata().supportsCastPredicatePushdown(connectorSession); + // A gateway's metadata policy covers the catalog. Its per-table scan provider can also + // reject CASTs before the expression converter unwraps them for file pruning. + if (supportsCast && scanProvider instanceof ConnectorPushdownOps) { + supportsCast = onPluginClassLoader(scanProvider, + () -> ((ConnectorPushdownOps) scanProvider).supportsCastPredicatePushdown(connectorSession)); + } for (int i = 0; i < conjuncts.size(); i++) { Expr conjunct = conjuncts.get(i); // The neutral converter strips CASTs. Instant casts can depend on the Doris session diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCastPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCastPredicateTest.java index 1d7252a43de2b4..cfb41c61648bcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCastPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeCastPredicateTest.java @@ -21,15 +21,19 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorPushdownOps; +import org.apache.doris.connector.spi.pushdown.ConnectorAnd; import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; import org.apache.doris.connector.spi.pushdown.ConnectorComparison; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; @@ -74,7 +78,7 @@ public void castConjunctsStayInDoris(String predicate) throws Exception { List before = Expr.cloneList(conjuncts); PluginDrivenScanNode node = nodeWith(conjuncts, false); - Optional filter = Deencapsulation.invoke(node, "buildRemainingFilter"); + Optional filter = buildFilter(node, Mockito.mock(ConnectorScanPlanProvider.class)); Assertions.assertFalse(filter.isPresent(), predicate); // Nothing was sent to the connector, so the source LIMIT must stay off ... @@ -93,7 +97,7 @@ public void independentPlainConjunctIsStillPushed() throws Exception { List before = Expr.cloneList(conjuncts); PluginDrivenScanNode node = nodeWith(conjuncts, false); - Optional filter = Deencapsulation.invoke(node, "buildRemainingFilter"); + Optional filter = buildFilter(node, Mockito.mock(ConnectorScanPlanProvider.class)); Assertions.assertTrue(filter.isPresent()); Assertions.assertTrue(filter.get() instanceof ConnectorComparison, filter.get().toString()); @@ -111,7 +115,7 @@ public void plainConjunctsArePushedWhole(String predicate) throws Exception { Assertions.assertFalse(conjuncts.stream().anyMatch(PluginDrivenScanNode::containsCastExpr), predicate); PluginDrivenScanNode node = nodeWith(conjuncts, false); - Optional filter = Deencapsulation.invoke(node, "buildRemainingFilter"); + Optional filter = buildFilter(node, Mockito.mock(ConnectorScanPlanProvider.class)); Assertions.assertTrue(filter.isPresent(), predicate); // Every conjunct went to the connector, so no index remapping is needed. @@ -123,13 +127,72 @@ public void castConjunctIsPushedWhenTheConnectorAcceptsCasts() throws Exception List conjuncts = planConjuncts("cast(code as int) = 5"); PluginDrivenScanNode node = nodeWith(conjuncts, true); - Optional filter = Deencapsulation.invoke(node, "buildRemainingFilter"); + Optional filter = buildFilter(node, Mockito.mock(ConnectorScanPlanProvider.class)); // The same conjunct is pushed once the connector accepts CASTs, so the rule above is the switch. Assertions.assertTrue(filter.isPresent()); Assertions.assertNull(Deencapsulation.getField(node, "filteredToOriginalIndex")); } + @ParameterizedTest + @CsvSource({"true,false", "false,true", "true,true"}) + public void tableScanCanVetoCastsWithoutRelaxingCatalogPolicy(boolean catalogAccepts, boolean readerAccepts) + throws Exception { + List conjuncts = planConjuncts("cast(amount as bigint) = 123"); + Assertions.assertTrue(conjuncts.stream().anyMatch(PluginDrivenScanNode::containsCastExpr)); + PluginDrivenScanNode node = nodeWith(conjuncts, catalogAccepts); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class, + Mockito.withSettings().extraInterfaces(ConnectorPushdownOps.class)); + Mockito.when(((ConnectorPushdownOps) provider).supportsCastPredicatePushdown(Mockito.any())) + .thenReturn(readerAccepts); + + Assertions.assertEquals(catalogAccepts && readerAccepts, buildFilter(node, provider).isPresent()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void sharedCatalogKeepsCastPolicyLocalToEachScan(boolean deltaFirst) throws Exception { + List conjuncts = planConjuncts("cast(amount as bigint) = 123 and id = 1"); + Assertions.assertEquals(2, conjuncts.size()); + Assertions.assertTrue(conjuncts.stream().anyMatch(PluginDrivenScanNode::containsCastExpr)); + ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); + Mockito.when(metadata.supportsCastPredicatePushdown(Mockito.any())).thenReturn(true); + // An HMS statement shares its catalog metadata across Delta and Iceberg scan nodes. + PluginDrivenScanNode delta = nodeWith(Expr.cloneList(conjuncts), metadata); + PluginDrivenScanNode iceberg = nodeWith(Expr.cloneList(conjuncts), metadata); + ConnectorScanPlanProvider deltaProvider = Mockito.mock(ConnectorScanPlanProvider.class, + Mockito.withSettings().extraInterfaces(ConnectorPushdownOps.class)); + Mockito.when(((ConnectorPushdownOps) deltaProvider).supportsCastPredicatePushdown(Mockito.any())) + .thenReturn(false); + ConnectorScanPlanProvider icebergProvider = Mockito.mock(ConnectorScanPlanProvider.class); + + // Revisit both scans in both orders: a catalog-wide or last-table policy must not leak between them. + List scans = deltaFirst ? List.of(delta, iceberg, delta, iceberg) + : List.of(iceberg, delta, iceberg, delta); + for (PluginDrivenScanNode scan : scans) { + ConnectorExpression filter = buildFilter(scan, scan == delta ? deltaProvider : icebergProvider) + .orElseThrow(); + if (scan == delta) { + Assertions.assertInstanceOf(ConnectorComparison.class, filter); + Assertions.assertEquals("id", + ((ConnectorColumnRef) ((ConnectorComparison) filter).getLeft()).getColumnName()); + int plainIndex = PluginDrivenScanNode.containsCastExpr(conjuncts.get(0)) ? 1 : 0; + Assertions.assertEquals(List.of(plainIndex), + Deencapsulation.getField(scan, "filteredToOriginalIndex")); + } else { + Assertions.assertInstanceOf(ConnectorAnd.class, filter); + Assertions.assertEquals(2, filter.getChildren().size()); + Assertions.assertNull(Deencapsulation.getField(scan, "filteredToOriginalIndex")); + } + Assertions.assertEquals(conjuncts, Deencapsulation.getField(scan, "conjuncts")); + } + } + + private static Optional buildFilter(PluginDrivenScanNode node, + ConnectorScanPlanProvider provider) { + return Deencapsulation.invoke(node, "buildRemainingFilter", provider); + } + private List planConjuncts(String predicate) throws Exception { StmtExecutor executor = new StmtExecutor(connectContext, "select * from db1.cast_predicates where " + predicate); @@ -144,9 +207,13 @@ private List planConjuncts(String predicate) throws Exception { * {@code cachedMetadata} is injected directly so the per-statement metadata funnel stays out of the test. */ private static PluginDrivenScanNode nodeWith(List conjuncts, boolean supportsCast) { - PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); Mockito.when(metadata.supportsCastPredicatePushdown(Mockito.any())).thenReturn(supportsCast); + return nodeWith(conjuncts, metadata); + } + + private static PluginDrivenScanNode nodeWith(List conjuncts, ConnectorMetadata metadata) { + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); Deencapsulation.setField(node, "cachedMetadata", metadata); Deencapsulation.setField(node, "conjuncts", conjuncts); return node; diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 9a9f99b68d55b5..3621bba13eefc1 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -486,6 +486,22 @@ struct THudiFileDesc { 12: optional i64 schema_id; // for schema change. (native reader) } +struct TDeltaLakeDeletionVectorDesc { + // On-disk DVs use an absolute path and an offset pointing at the length prefix. + 1: optional string path + 2: optional i64 offset + // Delta's sizeInBytes includes magic and bitmap, but excludes length and CRC. + 3: optional i32 size_in_bytes + 4: optional i64 cardinality + // Decoded inline DVs contain magic and bitmap only, without length or CRC. + 5: optional binary inline_data +} + +struct TDeltaLakeFileDesc { + 1: optional i64 schema_id + 2: optional TDeltaLakeDeletionVectorDesc deletion_vector +} + // Deprecated: the associated external table read support has been removed. struct TLakeSoulFileDesc { 1: optional list file_paths; @@ -754,6 +770,7 @@ struct TTableFormatFileDesc { // es_params, jdbc_params). 15: optional map fluss_params 16: optional TMongoScanDesc mongodb_params + 17: optional TDeltaLakeFileDesc deltalake_params } // Deprecated, hive text talbe is a special format, not a serde type diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 6b0c8eb3f99dad..c66d30da9fa111 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -143,6 +143,21 @@ oceanbase_cdc_port=2883 enableHiveTest=false enablePaimonTest=false +// Delta uses a separate Spark configuration, with Hive 3 HMS and MinIO. +enableDeltaLakeTest=false +// Optional synthetic planning fixture; see the Delta planning README. +enableDeltaLakePlanningTest=false +// Supply a local JSON file with warehouse and storage properties for each storage matrix run. +enableDeltaLakeStorageTest=false +// Optional SF1 TPC-H/TPC-DS comparison against the same Parquet files through Hive. +enableDeltaLakeBenchmarkTest=false +deltalakeBenchmarkQueryPattern=".*" +deltalakeStoragePropertiesFile="" +deltalakeMinioPort=19001 +deltalakeHmsPort=19383 +deltalakeSparkPort=11002 +deltalakeWarehouse="s3a://delta/warehouse" + // port of hive2 docker hive2HmsPort=9083 hive2HdfsPort=8020 diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_basic.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_basic.out new file mode 100644 index 00000000000000..c988106f8d4f5a --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_basic.out @@ -0,0 +1,139 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_tables -- +basic + +-- !filesystem_schema -- +id bigint Yes false \N +value text Yes false \N +part text Yes false \N + +-- !filesystem_true_basic -- +1 first a +2 second b +3 third \N + +-- !filesystem_true_count -- +3 + +-- !filesystem_true_null_partition -- +3 third \N + +-- !filesystem_true_one_partition -- +1 first a + +-- !filesystem_true_no_partition -- + +-- !filesystem_true_empty -- + +-- !filesystem_true_empty_count -- +0 + +-- !filesystem_true_types -- +1 true -128 32767 2147483647 9223372036854775807 1.25 -2.5 123456789012.3400 中文 / + % 00FF1D 2024-02-29 2024-03-10 02:30:00.123456+00:00 [1, null, 3] {"a":9, "b":10} {"mixed":7, "txt":"nested"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !filesystem_true_partition_types -- +1 true 42 12.50 2024-02-29 2024-03-10 02:30:00.123456+00:00 a/b=c%20+d 中文 +2 false -5 -9.00 1969-12-31 1969-12-31 23:59:59.123456+00:00 \N +3 \N \N \N \N \N \N + +-- !filesystem_byte_ranges -- +1 true -128 32767 2147483647 9223372036854775807 1.25 -2.5 123456789012.3400 中文 / + % 00FF1D 2024-02-29 2024-03-10 02:30:00.123456+00:00 [1, null, 3] {"a":9, "b":10} {"mixed":7, "txt":"nested"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !filesystem_UTC_timestamps -- +1 2024-03-10T02:30:00.123456 +2 \N + +-- !filesystem_UTC_partition_timestamps -- +1 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 +3 \N + +-- !filesystem_Asia_Shanghai_timestamps -- +1 2024-03-10T10:30:00.123456 +2 \N + +-- !filesystem_Asia_Shanghai_partition_timestamps -- +1 2024-03-10T10:30:00.123456 +2 1970-01-01T07:59:59.123456 +3 \N + +-- !filesystem_America_Los_Angeles_timestamps -- +1 2024-03-09T18:30:00.123456 +2 \N + +-- !filesystem_America_Los_Angeles_partition_timestamps -- +1 2024-03-09T18:30:00.123456 +2 1969-12-31T15:59:59.123456 +3 \N + +-- !hms_tables -- +basic + +-- !hms_schema -- +id bigint Yes false \N +value text Yes false \N +part text Yes false \N + +-- !hms_true_basic -- +1 first a +2 second b +3 third \N + +-- !hms_true_count -- +3 + +-- !hms_true_null_partition -- +3 third \N + +-- !hms_true_one_partition -- +1 first a + +-- !hms_true_no_partition -- + +-- !hms_true_empty -- + +-- !hms_true_empty_count -- +0 + +-- !hms_true_types -- +1 true -128 32767 2147483647 9223372036854775807 1.25 -2.5 123456789012.3400 中文 / + % 00FF1D 2024-02-29 2024-03-10 02:30:00.123456+00:00 [1, null, 3] {"a":9, "b":10} {"mixed":7, "txt":"nested"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !hms_true_partition_types -- +1 true 42 12.50 2024-02-29 2024-03-10 02:30:00.123456+00:00 a/b=c%20+d 中文 +2 false -5 -9.00 1969-12-31 1969-12-31 23:59:59.123456+00:00 \N +3 \N \N \N \N \N \N + +-- !hms_byte_ranges -- +1 true -128 32767 2147483647 9223372036854775807 1.25 -2.5 123456789012.3400 中文 / + % 00FF1D 2024-02-29 2024-03-10 02:30:00.123456+00:00 [1, null, 3] {"a":9, "b":10} {"mixed":7, "txt":"nested"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !hms_UTC_timestamps -- +1 2024-03-10T02:30:00.123456 +2 \N + +-- !hms_UTC_partition_timestamps -- +1 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 +3 \N + +-- !hms_Asia_Shanghai_timestamps -- +1 2024-03-10T10:30:00.123456 +2 \N + +-- !hms_Asia_Shanghai_partition_timestamps -- +1 2024-03-10T10:30:00.123456 +2 1970-01-01T07:59:59.123456 +3 \N + +-- !hms_America_Los_Angeles_timestamps -- +1 2024-03-09T18:30:00.123456 +2 \N + +-- !hms_America_Los_Angeles_partition_timestamps -- +1 2024-03-09T18:30:00.123456 +2 1969-12-31T15:59:59.123456 +3 \N + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_cast_pruning.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_cast_pruning.out new file mode 100644 index 00000000000000..2bf8d55b2309ec --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_cast_pruning.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_sync_cast -- +123.45 123 + +-- !filesystem_sync_count -- +1 + +-- !filesystem_sync_try_or -- +123.45 + +-- !filesystem_sync_and -- +123.45 + +-- !filesystem_stream_cast -- +123.45 123 + +-- !filesystem_stream_count -- +1 + +-- !filesystem_stream_try_or -- +123.45 + +-- !filesystem_stream_and -- +123.45 + +-- !hms_sync_cast -- +123.45 123 + +-- !hms_sync_count -- +1 + +-- !hms_sync_try_or -- +123.45 + +-- !hms_sync_and -- +123.45 + +-- !hms_stream_cast -- +123.45 123 + +-- !hms_stream_count -- +1 + +-- !hms_stream_try_or -- +123.45 + +-- !hms_stream_and -- +123.45 + +-- !gateway_sync_cast -- +123.45 123 + +-- !gateway_sync_count -- +1 + +-- !gateway_sync_try_or -- +123.45 + +-- !gateway_sync_and -- +123.45 + +-- !gateway_stream_cast -- +123.45 123 + +-- !gateway_stream_count -- +1 + +-- !gateway_stream_try_or -- +123.45 + +-- !gateway_stream_and -- +123.45 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_features.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_features.out new file mode 100644 index 00000000000000..8d080b526b9087 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_features.out @@ -0,0 +1,1193 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_name_schema -- +renamed bigint Yes false \N +nested struct Yes false \N +arr array> Yes false \N +m map> Yes false \N +partition_renamed text Yes false \N +old text Yes false \N + +-- !filesystem_name_all -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_name_filter -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !filesystem_name_partition -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !filesystem_name_null_partition -- +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_name_nested -- +1 10 \N 100 1000 + +-- !filesystem_name_golden -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !filesystem_id_schema -- +renamed bigint Yes false \N +nested struct Yes false \N +arr array> Yes false \N +m map> Yes false \N +partition_renamed text Yes false \N +old text Yes false \N + +-- !filesystem_id_all -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_id_filter -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !filesystem_id_partition -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !filesystem_id_null_partition -- +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_id_nested -- +1 10 \N 100 1000 + +-- !filesystem_id_golden -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !filesystem_UTC_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !filesystem_UTC_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !filesystem_UTC_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_UTC_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_UTC_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_Asia_Shanghai_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !filesystem_Asia_Shanghai_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !filesystem_Asia_Shanghai_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_Asia_Shanghai_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_Asia_Shanghai_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_America_Los_Angeles_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !filesystem_America_Los_Angeles_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !filesystem_America_Los_Angeles_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_America_Los_Angeles_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_America_Los_Angeles_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !filesystem_widening -- +1 -128 32767 2147483647 -2147483648 123456789.0000 1.25 1234.5600 2024-02-29T00:00 {"s":7} [8] {9:10} +2 128 2147483648 2147483648 1.5 123.4567 2.5 1.2345 2024-02-29T12:34:56.123456 {"s":2147483648} [2147483648] {2147483648:2147483648} + +-- !filesystem_widening_filter -- +2 128 2147483648 2147483648 1.5 123.4567 2.5 1.2345 2024-02-29T12:34:56.123456 {"s":2147483648} [2147483648] {2147483648:2147483648} + +-- !filesystem_golden_widening -- +1 2 3.4000000953674316 5 6 7 123.45 67.89000 1.0 2.0 3.0 4.0 2024-09-09T00:00 +9223372036854775807 9223372036854775807 1.234567890123 1.234567890123 1.234567890123 1.234567890123 12345678901234.56 12345678901.23456 123.4 12345.6 1234567890.1 123456789012345678.9 2024-09-09T12:34:56.123456 + +-- !filesystem_golden_nested_widening -- +{"a":1} {2:3} [4, 5] +{"a":9223372036854775807} {9223372036854775807:9223372036854775807} [9223372036854775807, -9223372036854775808] + +-- !filesystem_true_dv_count -- +85 + +-- !filesystem_true_dv_column_count -- +85 + +-- !filesystem_true_dv_filtered_count -- +9 + +-- !filesystem_true_dv_partitioned_with_checkpoint_count -- +35 + +-- !filesystem_true_dv_with_columnmapping_count -- +35 + +-- !filesystem_true_log_replay_dv_key_cases_count -- +47 + +-- !filesystem_false_dv_count -- +85 + +-- !filesystem_false_dv_column_count -- +85 + +-- !filesystem_false_dv_filtered_count -- +9 + +-- !filesystem_false_dv_partitioned_with_checkpoint_count -- +35 + +-- !filesystem_false_dv_with_columnmapping_count -- +35 + +-- !filesystem_false_log_replay_dv_key_cases_count -- +47 + +-- !filesystem_true_dv_rows -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !filesystem_true_dv_deleted -- + +-- !filesystem_true_dv_partition -- +1 merged-1 1 +11 updated-11 1 +13 value-13 1 +15 value-15 1 +17 value-17 1 +19 value-19 1 +3 value-3 1 +5 value-5 1 +9 value-9 1 + +-- !filesystem_true_dv_updates -- +11 updated-11 1 +22 updated-22 0 +33 updated-33 1 +44 updated-44 0 +55 updated-55 1 +66 updated-66 0 +88 updated-88 0 +99 updated-99 1 + +-- !filesystem_true_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_true_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_true_log_replay_dv_key_cases -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +4 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +5 +6 +8 +9 + +-- !filesystem_dv_byte_ranges -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !filesystem_mapping_byte_ranges -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_golden_dv_byte_ranges -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_name_schema -- +renamed bigint Yes false \N +nested struct Yes false \N +arr array> Yes false \N +m map> Yes false \N +partition_renamed text Yes false \N +old text Yes false \N + +-- !hms_name_all -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_name_filter -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !hms_name_partition -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !hms_name_null_partition -- +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_name_nested -- +1 10 \N 100 1000 + +-- !hms_name_golden -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !hms_id_schema -- +renamed bigint Yes false \N +nested struct Yes false \N +arr array> Yes false \N +m map> Yes false \N +partition_renamed text Yes false \N +old text Yes false \N + +-- !hms_id_all -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_id_filter -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !hms_id_partition -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !hms_id_null_partition -- +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_id_nested -- +1 10 \N 100 1000 + +-- !hms_id_golden -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !hms_UTC_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !hms_UTC_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !hms_UTC_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_UTC_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_UTC_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_Asia_Shanghai_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !hms_Asia_Shanghai_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !hms_Asia_Shanghai_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_Asia_Shanghai_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_Asia_Shanghai_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_America_Los_Angeles_ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !hms_America_Los_Angeles_ntz_filter -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 + +-- !hms_America_Los_Angeles_golden_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_America_Los_Angeles_golden_ntz_name_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_America_Los_Angeles_golden_ntz_id_mode -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !hms_widening -- +1 -128 32767 2147483647 -2147483648 123456789.0000 1.25 1234.5600 2024-02-29T00:00 {"s":7} [8] {9:10} +2 128 2147483648 2147483648 1.5 123.4567 2.5 1.2345 2024-02-29T12:34:56.123456 {"s":2147483648} [2147483648] {2147483648:2147483648} + +-- !hms_widening_filter -- +2 128 2147483648 2147483648 1.5 123.4567 2.5 1.2345 2024-02-29T12:34:56.123456 {"s":2147483648} [2147483648] {2147483648:2147483648} + +-- !hms_golden_widening -- +1 2 3.4000000953674316 5 6 7 123.45 67.89000 1.0 2.0 3.0 4.0 2024-09-09T00:00 +9223372036854775807 9223372036854775807 1.234567890123 1.234567890123 1.234567890123 1.234567890123 12345678901234.56 12345678901.23456 123.4 12345.6 1234567890.1 123456789012345678.9 2024-09-09T12:34:56.123456 + +-- !hms_golden_nested_widening -- +{"a":1} {2:3} [4, 5] +{"a":9223372036854775807} {9223372036854775807:9223372036854775807} [9223372036854775807, -9223372036854775808] + +-- !hms_true_dv_count -- +85 + +-- !hms_true_dv_column_count -- +85 + +-- !hms_true_dv_filtered_count -- +9 + +-- !hms_true_dv_partitioned_with_checkpoint_count -- +35 + +-- !hms_true_dv_with_columnmapping_count -- +35 + +-- !hms_true_log_replay_dv_key_cases_count -- +47 + +-- !hms_false_dv_count -- +85 + +-- !hms_false_dv_column_count -- +85 + +-- !hms_false_dv_filtered_count -- +9 + +-- !hms_false_dv_partitioned_with_checkpoint_count -- +35 + +-- !hms_false_dv_with_columnmapping_count -- +35 + +-- !hms_false_log_replay_dv_key_cases_count -- +47 + +-- !hms_true_dv_rows -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !hms_true_dv_deleted -- + +-- !hms_true_dv_partition -- +1 merged-1 1 +11 updated-11 1 +13 value-13 1 +15 value-15 1 +17 value-17 1 +19 value-19 1 +3 value-3 1 +5 value-5 1 +9 value-9 1 + +-- !hms_true_dv_updates -- +11 updated-11 1 +22 updated-22 0 +33 updated-33 1 +44 updated-44 0 +55 updated-55 1 +66 updated-66 0 +88 updated-88 0 +99 updated-99 1 + +-- !hms_true_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_true_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_true_log_replay_dv_key_cases -- +1 +10 +11 +12 +13 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +4 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +5 +6 +8 +9 + +-- !hms_dv_byte_ranges -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !hms_mapping_byte_ranges -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_golden_dv_byte_ranges -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_hive.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_hive.out new file mode 100644 index 00000000000000..964025ed4b3e96 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_hive.out @@ -0,0 +1,301 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mapping_schema -- +renamed bigint Yes false \N +nested struct Yes false \N +arr array> Yes false \N +m map> Yes false \N +partition_renamed text Yes false \N +old text Yes false \N + +-- !mapping_ddl -- +mapping_id CREATE TABLE `mapping_id` (\n `renamed` bigint NULL,\n `nested` struct NULL,\n `arr` array> NULL,\n `m` map> NULL,\n `partition_renamed` text NULL,\n `old` text NULL\n) ENGINE=hms; + +-- !mapping_name -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !mapping_name_count -- +2 + +-- !mapping_id -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !mapping_id_count -- +2 + +-- !ntz -- +1 2024-03-10T02:30:00.123456 2024-03-10T02:30:00.123456 +2 1969-12-31T23:59:59.123456 \N + +-- !ntz_count -- +2 + +-- !widening -- +1 -128 32767 2147483647 -2147483648 123456789.0000 1.25 1234.5600 2024-02-29T00:00 {"s":7} [8] {9:10} +2 128 2147483648 2147483648 1.5 123.4567 2.5 1.2345 2024-02-29T12:34:56.123456 {"s":2147483648} [2147483648] {2147483648:2147483648} + +-- !widening_count -- +2 + +-- !dv -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !dv_count -- +85 + +-- !log_compaction -- +1 101 +20 200 +21 210 +22 220 +3 3 +5 5 +7 7 +9 9 + +-- !log_compaction_count -- +8 + +-- !mapping_filter -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N + +-- !dv_filter -- +1 merged-1 1 +11 updated-11 1 +13 value-13 1 +15 value-15 1 +17 value-17 1 +19 value-19 1 +3 value-3 1 +5 value-5 1 +9 value-9 1 + +-- !v2_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !v2_parquet -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !dv_streaming -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !mapping_streaming -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !initial -- +1 + +-- !cached -- +1 + +-- !table_refresh -- +1 +2 + +-- !refreshed_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !schema_refresh -- +1 \N +2 \N +3 catalog-refresh + +-- !database_refresh -- +1 \N +2 \N +3 catalog-refresh +4 database-refresh + +-- !plain_hive -- +1 hive +2 parquet + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.out new file mode 100644 index 00000000000000..6c44384ca278ad --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.out @@ -0,0 +1,50 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !iceberg_fixture -- +123 123.00 +999 999.00 + +-- !iceberg_files -- +2 + +-- !sync_delta_first -- +delta 123.45 +iceberg 123.00 + +-- !sync_iceberg_first -- +delta 123.45 +iceberg 123.00 + +-- !sync_plain_and -- +delta 123.45 +iceberg 123.00 + +-- !sync_join_delta_first -- +123.45 123 123.00 + +-- !sync_join_iceberg_first -- +123.45 123 123.00 + +-- !sync_iceberg_after -- +iceberg 123.00 + +-- !stream_delta_first -- +delta 123.45 +iceberg 123.00 + +-- !stream_iceberg_first -- +delta 123.45 +iceberg 123.00 + +-- !stream_plain_and -- +delta 123.45 +iceberg 123.00 + +-- !stream_join_delta_first -- +123.45 123 123.00 + +-- !stream_join_iceberg_first -- +123.45 123 123.00 + +-- !stream_iceberg_after -- +iceberg 123.00 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_registration.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_registration.out new file mode 100644 index 00000000000000..168389a015b4b2 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_hms_registration.out @@ -0,0 +1,487 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dedicated_databases -- +delta_hms_reg +delta_hms_reg_dropped +delta_hms_reg_plain + +-- !dedicated_tables -- +reg_ctas +reg_drop_external +reg_drop_managed +reg_external +reg_managed +reg_mixed_case +reg_no_columns +reg_recreate +reg_rename_external +reg_rename_managed +reg_replace +reg_space_path +reg_trailing_slash + +-- !dedicated_plain_db_tables -- + +-- !dedicated_info_tables -- +reg_managed +reg_mixed_case + +-- !dedicated_info_columns -- +id bigint +part varchar +value varchar + +-- !dedicated_reg_managed_schema -- +id bigint Yes false \N +value text Yes false \N +part text Yes false \N + +-- !dedicated_reg_managed -- +1 one a +3 three b +4 four \N + +-- !dedicated_reg_managed_count -- +3 + +-- !dedicated_reg_ctas_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !dedicated_reg_ctas -- +1 ctas-one +2 ctas-two + +-- !dedicated_reg_ctas_count -- +2 + +-- !dedicated_reg_external_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !dedicated_reg_external -- +1 external-one +2 external-two + +-- !dedicated_reg_external_count -- +2 + +-- !dedicated_reg_no_columns_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !dedicated_reg_no_columns -- +1 external-one +2 external-two + +-- !dedicated_reg_no_columns_count -- +2 + +-- !dedicated_reg_space_path_schema -- +id bigint Yes false \N + +-- !dedicated_reg_space_path -- +1 + +-- !dedicated_reg_space_path_count -- +1 + +-- !dedicated_reg_trailing_slash_schema -- +id bigint Yes false \N + +-- !dedicated_reg_trailing_slash -- +1 + +-- !dedicated_reg_trailing_slash_count -- +1 + +-- !dedicated_reg_mixed_case_schema -- +ID bigint Yes false \N +MixedValue text Yes false \N + +-- !dedicated_reg_mixed_case -- +1 mixed + +-- !dedicated_reg_mixed_case_count -- +1 + +-- !dedicated_external_ddl -- +reg_external CREATE TABLE `reg_external` (\n `id` bigint NULL,\n `value` text NULL\n) ENGINE=Delta Lake; + +-- !dedicated_managed_partition -- +1 one a + +-- !dedicated_managed_null_partition -- +4 four \N + +-- !dedicated_mixed_case_column -- +1 mixed + +-- !dedicated_external_after_rejected_writes -- +1 external-one +2 external-two + +-- !dedicated_reg_rename_external_initial -- +1 before +2 before + +-- !dedicated_reg_rename_managed_initial -- +1 before +2 before + +-- !dedicated_reg_drop_external_initial -- +1 + +-- !dedicated_reg_drop_managed_initial -- +1 + +-- !dedicated_reg_recreate_initial -- +1 + +-- !dedicated_reg_replace_initial -- +1 + +-- !dedicated_in_dropped_db_initial -- +1 + +-- !gateway_databases -- +delta_hms_reg +delta_hms_reg_dropped +delta_hms_reg_plain + +-- !gateway_tables -- +reg_ctas +reg_drop_external +reg_drop_managed +reg_external +reg_managed +reg_mixed_case +reg_no_columns +reg_plain +reg_recreate +reg_rename_external +reg_rename_managed +reg_replace +reg_space_path +reg_trailing_slash +reg_view + +-- !gateway_plain_db_tables -- +plain_only + +-- !gateway_info_tables -- +reg_managed +reg_mixed_case + +-- !gateway_info_columns -- +id bigint +part varchar +value varchar + +-- !gateway_reg_managed_schema -- +id bigint Yes false \N +value text Yes false \N +part text Yes false \N + +-- !gateway_reg_managed -- +1 one a +3 three b +4 four \N + +-- !gateway_reg_managed_count -- +3 + +-- !gateway_reg_ctas_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !gateway_reg_ctas -- +1 ctas-one +2 ctas-two + +-- !gateway_reg_ctas_count -- +2 + +-- !gateway_reg_external_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !gateway_reg_external -- +1 external-one +2 external-two + +-- !gateway_reg_external_count -- +2 + +-- !gateway_reg_no_columns_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !gateway_reg_no_columns -- +1 external-one +2 external-two + +-- !gateway_reg_no_columns_count -- +2 + +-- !gateway_reg_space_path_schema -- +id bigint Yes false \N + +-- !gateway_reg_space_path -- +1 + +-- !gateway_reg_space_path_count -- +1 + +-- !gateway_reg_trailing_slash_schema -- +id bigint Yes false \N + +-- !gateway_reg_trailing_slash -- +1 + +-- !gateway_reg_trailing_slash_count -- +1 + +-- !gateway_reg_mixed_case_schema -- +ID bigint Yes false \N +MixedValue text Yes false \N + +-- !gateway_reg_mixed_case -- +1 mixed + +-- !gateway_reg_mixed_case_count -- +1 + +-- !gateway_external_ddl -- +reg_external CREATE TABLE `reg_external` (\n `id` bigint NULL,\n `value` text NULL\n) ENGINE=hms; + +-- !gateway_managed_partition -- +1 one a + +-- !gateway_managed_null_partition -- +4 four \N + +-- !gateway_mixed_case_column -- +1 mixed + +-- !gateway_plain -- +1 parquet + +-- !gateway_external_after_rejected_writes -- +1 external-one +2 external-two + +-- !gateway_reg_rename_external_initial -- +1 before +2 before + +-- !gateway_reg_rename_managed_initial -- +1 before +2 before + +-- !gateway_reg_drop_external_initial -- +1 + +-- !gateway_reg_drop_managed_initial -- +1 + +-- !gateway_reg_recreate_initial -- +1 + +-- !gateway_reg_replace_initial -- +1 + +-- !gateway_in_dropped_db_initial -- +1 + +-- !dedicated_case_0_lower -- +1 mixed + +-- !dedicated_case_0_tables -- + +-- !dedicated_case_1_lower -- +1 mixed + +-- !dedicated_case_1_tables -- +reg_mixed_case + +-- !dedicated_case_1_upper -- +1 mixed + +-- !dedicated_case_2_lower -- +1 mixed + +-- !dedicated_case_2_tables -- +reg_mixed_case + +-- !dedicated_case_2_upper -- +1 mixed + +-- !gateway_case_0_lower -- +1 mixed + +-- !gateway_case_0_tables -- + +-- !gateway_case_1_lower -- +1 mixed + +-- !gateway_case_1_tables -- +reg_mixed_case + +-- !gateway_case_1_upper -- +1 mixed + +-- !gateway_case_2_lower -- +1 mixed + +-- !gateway_case_2_tables -- +reg_mixed_case + +-- !gateway_case_2_upper -- +1 mixed + +-- !dedicated_tables_after_changes -- +reg_added_later +reg_ctas +reg_external +reg_managed +reg_mixed_case +reg_no_columns +reg_reattached +reg_recreate +reg_rename_external_new +reg_rename_managed_new +reg_replace +reg_space_path +reg_trailing_slash + +-- !dedicated_reg_rename_external_new_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !dedicated_reg_rename_external_new -- +2 before +3 after + +-- !dedicated_reg_rename_external_new_count -- +2 + +-- !dedicated_reg_rename_managed_new_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !dedicated_reg_rename_managed_new -- +2 before +3 after + +-- !dedicated_reg_rename_managed_new_count -- +2 + +-- !dedicated_reattached_schema -- +id bigint Yes false \N + +-- !dedicated_reattached -- +1 + +-- !dedicated_reg_recreate_schema_after_change -- +recreated text Yes false \N + +-- !dedicated_reg_recreate_after_change -- +recreated + +-- !dedicated_reg_replace_schema_after_change -- +replaced text Yes false \N + +-- !dedicated_reg_replace_after_change -- +replaced + +-- !dedicated_reg_external_shared_path -- +1 external-one +2 external-two +3 external-three + +-- !dedicated_reg_no_columns_shared_path -- +1 external-one +2 external-two +3 external-three + +-- !dedicated_added_later -- +1 + +-- !dedicated_databases_after_drop -- +delta_hms_reg +delta_hms_reg_plain + +-- !gateway_tables_after_changes -- +reg_added_later +reg_ctas +reg_external +reg_managed +reg_mixed_case +reg_no_columns +reg_plain +reg_reattached +reg_recreate +reg_rename_external_new +reg_rename_managed_new +reg_replace +reg_space_path +reg_trailing_slash +reg_view + +-- !gateway_reg_rename_external_new_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !gateway_reg_rename_external_new -- +2 before +3 after + +-- !gateway_reg_rename_external_new_count -- +2 + +-- !gateway_reg_rename_managed_new_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !gateway_reg_rename_managed_new -- +2 before +3 after + +-- !gateway_reg_rename_managed_new_count -- +2 + +-- !gateway_reattached_schema -- +id bigint Yes false \N + +-- !gateway_reattached -- +1 + +-- !gateway_reg_recreate_schema_after_change -- +recreated text Yes false \N + +-- !gateway_reg_recreate_after_change -- +recreated + +-- !gateway_reg_replace_schema_after_change -- +replaced text Yes false \N + +-- !gateway_reg_replace_after_change -- +replaced + +-- !gateway_reg_external_shared_path -- +1 external-one +2 external-two +3 external-three + +-- !gateway_reg_no_columns_shared_path -- +1 external-one +2 external-two +3 external-three + +-- !gateway_added_later -- +1 + +-- !gateway_databases_after_drop -- +delta_hms_reg +delta_hms_reg_plain + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_metadata.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_metadata.out new file mode 100644 index 00000000000000..113dc3ba91af25 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_metadata.out @@ -0,0 +1,1647 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_false_multi_part_checkpoint -- +0 +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_false_multi_part_checkpoint_count -- +31 + +-- !filesystem_false_only_checkpoint_files -- +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +5 +6 +6 +7 +7 +8 +8 +9 +9 + +-- !filesystem_false_only_checkpoint_files_count -- +25 + +-- !filesystem_false_v2_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_false_v2_checkpoint_json_count -- +10 + +-- !filesystem_false_v2_checkpoint_parquet -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_false_v2_checkpoint_parquet_count -- +10 + +-- !filesystem_false_basic_with_vacuum_protocol_check_feature -- +0 val=0 +1 val=1 +10 val=10 +11 val=11 +12 val=12 +13 val=13 +14 val=14 +15 val=15 +16 val=16 +17 val=17 +18 val=18 +19 val=19 +2 val=2 +20 val=20 +21 val=21 +22 val=22 +23 val=23 +24 val=24 +25 val=25 +26 val=26 +27 val=27 +28 val=28 +29 val=29 +3 val=3 +30 val=30 +31 val=31 +32 val=32 +33 val=33 +34 val=34 +35 val=35 +36 val=36 +37 val=37 +38 val=38 +39 val=39 +4 val=4 +40 val=40 +41 val=41 +42 val=42 +43 val=43 +44 val=44 +45 val=45 +46 val=46 +47 val=47 +48 val=48 +49 val=49 +5 val=5 +50 val=50 +51 val=51 +52 val=52 +53 val=53 +54 val=54 +55 val=55 +56 val=56 +57 val=57 +58 val=58 +59 val=59 +6 val=6 +60 val=60 +61 val=61 +62 val=62 +63 val=63 +64 val=64 +65 val=65 +66 val=66 +67 val=67 +68 val=68 +69 val=69 +7 val=7 +70 val=70 +71 val=71 +72 val=72 +73 val=73 +74 val=74 +75 val=75 +76 val=76 +77 val=77 +78 val=78 +79 val=79 +8 val=8 +80 val=80 +81 val=81 +82 val=82 +83 val=83 +84 val=84 +85 val=85 +86 val=86 +87 val=87 +88 val=88 +89 val=89 +9 val=9 +90 val=90 +91 val=91 +92 val=92 +93 val=93 +94 val=94 +95 val=95 +96 val=96 +97 val=97 +98 val=98 +99 val=99 + +-- !filesystem_false_basic_with_vacuum_protocol_check_feature_count -- +100 + +-- !filesystem_false_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_false_dv_partitioned_with_checkpoint_count -- +35 + +-- !filesystem_false_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_false_dv_with_columnmapping_count -- +35 + +-- !filesystem_false_compaction -- +1 101 +20 200 +21 210 +22 220 +3 3 +5 5 +7 7 +9 9 + +-- !filesystem_false_pruning -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !filesystem_false_count -- +21 + +-- !filesystem_false_dv_splits -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !filesystem_false_mapping_splits -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_true_multi_part_checkpoint -- +0 +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_true_multi_part_checkpoint_count -- +31 + +-- !filesystem_true_only_checkpoint_files -- +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +5 +6 +6 +7 +7 +8 +8 +9 +9 + +-- !filesystem_true_only_checkpoint_files_count -- +25 + +-- !filesystem_true_v2_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_true_v2_checkpoint_json_count -- +10 + +-- !filesystem_true_v2_checkpoint_parquet -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !filesystem_true_v2_checkpoint_parquet_count -- +10 + +-- !filesystem_true_basic_with_vacuum_protocol_check_feature -- +0 val=0 +1 val=1 +10 val=10 +11 val=11 +12 val=12 +13 val=13 +14 val=14 +15 val=15 +16 val=16 +17 val=17 +18 val=18 +19 val=19 +2 val=2 +20 val=20 +21 val=21 +22 val=22 +23 val=23 +24 val=24 +25 val=25 +26 val=26 +27 val=27 +28 val=28 +29 val=29 +3 val=3 +30 val=30 +31 val=31 +32 val=32 +33 val=33 +34 val=34 +35 val=35 +36 val=36 +37 val=37 +38 val=38 +39 val=39 +4 val=4 +40 val=40 +41 val=41 +42 val=42 +43 val=43 +44 val=44 +45 val=45 +46 val=46 +47 val=47 +48 val=48 +49 val=49 +5 val=5 +50 val=50 +51 val=51 +52 val=52 +53 val=53 +54 val=54 +55 val=55 +56 val=56 +57 val=57 +58 val=58 +59 val=59 +6 val=6 +60 val=60 +61 val=61 +62 val=62 +63 val=63 +64 val=64 +65 val=65 +66 val=66 +67 val=67 +68 val=68 +69 val=69 +7 val=7 +70 val=70 +71 val=71 +72 val=72 +73 val=73 +74 val=74 +75 val=75 +76 val=76 +77 val=77 +78 val=78 +79 val=79 +8 val=8 +80 val=80 +81 val=81 +82 val=82 +83 val=83 +84 val=84 +85 val=85 +86 val=86 +87 val=87 +88 val=88 +89 val=89 +9 val=9 +90 val=90 +91 val=91 +92 val=92 +93 val=93 +94 val=94 +95 val=95 +96 val=96 +97 val=97 +98 val=98 +99 val=99 + +-- !filesystem_true_basic_with_vacuum_protocol_check_feature_count -- +100 + +-- !filesystem_true_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_true_dv_partitioned_with_checkpoint_count -- +35 + +-- !filesystem_true_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !filesystem_true_dv_with_columnmapping_count -- +35 + +-- !filesystem_true_compaction -- +1 101 +20 200 +21 210 +22 220 +3 3 +5 5 +7 7 +9 9 + +-- !filesystem_true_pruning -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !filesystem_true_count -- +21 + +-- !filesystem_true_dv_splits -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !filesystem_true_mapping_splits -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_initial -- +1 + +-- !filesystem_cached -- +1 + +-- !filesystem_uncached -- +1 +2 + +-- !filesystem_table_refresh -- +1 +2 + +-- !filesystem_cached_schema -- +id bigint Yes false \N + +-- !filesystem_refreshed_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !filesystem_schema_rows -- +1 \N +2 \N +3 new-schema + +-- !filesystem_database_refresh -- +1 \N +2 \N +3 new-schema +4 database-refresh + +-- !filesystem_catalog_refresh -- +1 \N +2 \N +3 new-schema +4 database-refresh +5 catalog-refresh + +-- !hms_false_multi_part_checkpoint -- +0 +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_false_multi_part_checkpoint_count -- +31 + +-- !hms_false_only_checkpoint_files -- +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +5 +6 +6 +7 +7 +8 +8 +9 +9 + +-- !hms_false_only_checkpoint_files_count -- +25 + +-- !hms_false_v2_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_false_v2_checkpoint_json_count -- +10 + +-- !hms_false_v2_checkpoint_parquet -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_false_v2_checkpoint_parquet_count -- +10 + +-- !hms_false_basic_with_vacuum_protocol_check_feature -- +0 val=0 +1 val=1 +10 val=10 +11 val=11 +12 val=12 +13 val=13 +14 val=14 +15 val=15 +16 val=16 +17 val=17 +18 val=18 +19 val=19 +2 val=2 +20 val=20 +21 val=21 +22 val=22 +23 val=23 +24 val=24 +25 val=25 +26 val=26 +27 val=27 +28 val=28 +29 val=29 +3 val=3 +30 val=30 +31 val=31 +32 val=32 +33 val=33 +34 val=34 +35 val=35 +36 val=36 +37 val=37 +38 val=38 +39 val=39 +4 val=4 +40 val=40 +41 val=41 +42 val=42 +43 val=43 +44 val=44 +45 val=45 +46 val=46 +47 val=47 +48 val=48 +49 val=49 +5 val=5 +50 val=50 +51 val=51 +52 val=52 +53 val=53 +54 val=54 +55 val=55 +56 val=56 +57 val=57 +58 val=58 +59 val=59 +6 val=6 +60 val=60 +61 val=61 +62 val=62 +63 val=63 +64 val=64 +65 val=65 +66 val=66 +67 val=67 +68 val=68 +69 val=69 +7 val=7 +70 val=70 +71 val=71 +72 val=72 +73 val=73 +74 val=74 +75 val=75 +76 val=76 +77 val=77 +78 val=78 +79 val=79 +8 val=8 +80 val=80 +81 val=81 +82 val=82 +83 val=83 +84 val=84 +85 val=85 +86 val=86 +87 val=87 +88 val=88 +89 val=89 +9 val=9 +90 val=90 +91 val=91 +92 val=92 +93 val=93 +94 val=94 +95 val=95 +96 val=96 +97 val=97 +98 val=98 +99 val=99 + +-- !hms_false_basic_with_vacuum_protocol_check_feature_count -- +100 + +-- !hms_false_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_false_dv_partitioned_with_checkpoint_count -- +35 + +-- !hms_false_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_false_dv_with_columnmapping_count -- +35 + +-- !hms_false_compaction -- +1 101 +20 200 +21 210 +22 220 +3 3 +5 5 +7 7 +9 9 + +-- !hms_false_pruning -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !hms_false_count -- +21 + +-- !hms_false_dv_splits -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !hms_false_mapping_splits -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_true_multi_part_checkpoint -- +0 +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_true_multi_part_checkpoint_count -- +31 + +-- !hms_true_only_checkpoint_files -- +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +3 +4 +5 +5 +6 +6 +7 +7 +8 +8 +9 +9 + +-- !hms_true_only_checkpoint_files_count -- +25 + +-- !hms_true_v2_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_true_v2_checkpoint_json_count -- +10 + +-- !hms_true_v2_checkpoint_parquet -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !hms_true_v2_checkpoint_parquet_count -- +10 + +-- !hms_true_basic_with_vacuum_protocol_check_feature -- +0 val=0 +1 val=1 +10 val=10 +11 val=11 +12 val=12 +13 val=13 +14 val=14 +15 val=15 +16 val=16 +17 val=17 +18 val=18 +19 val=19 +2 val=2 +20 val=20 +21 val=21 +22 val=22 +23 val=23 +24 val=24 +25 val=25 +26 val=26 +27 val=27 +28 val=28 +29 val=29 +3 val=3 +30 val=30 +31 val=31 +32 val=32 +33 val=33 +34 val=34 +35 val=35 +36 val=36 +37 val=37 +38 val=38 +39 val=39 +4 val=4 +40 val=40 +41 val=41 +42 val=42 +43 val=43 +44 val=44 +45 val=45 +46 val=46 +47 val=47 +48 val=48 +49 val=49 +5 val=5 +50 val=50 +51 val=51 +52 val=52 +53 val=53 +54 val=54 +55 val=55 +56 val=56 +57 val=57 +58 val=58 +59 val=59 +6 val=6 +60 val=60 +61 val=61 +62 val=62 +63 val=63 +64 val=64 +65 val=65 +66 val=66 +67 val=67 +68 val=68 +69 val=69 +7 val=7 +70 val=70 +71 val=71 +72 val=72 +73 val=73 +74 val=74 +75 val=75 +76 val=76 +77 val=77 +78 val=78 +79 val=79 +8 val=8 +80 val=80 +81 val=81 +82 val=82 +83 val=83 +84 val=84 +85 val=85 +86 val=86 +87 val=87 +88 val=88 +89 val=89 +9 val=9 +90 val=90 +91 val=91 +92 val=92 +93 val=93 +94 val=94 +95 val=95 +96 val=96 +97 val=97 +98 val=98 +99 val=99 + +-- !hms_true_basic_with_vacuum_protocol_check_feature_count -- +100 + +-- !hms_true_dv_partitioned_with_checkpoint -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_true_dv_partitioned_with_checkpoint_count -- +35 + +-- !hms_true_dv_with_columnmapping -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !hms_true_dv_with_columnmapping_count -- +35 + +-- !hms_true_compaction -- +1 101 +20 200 +21 210 +22 220 +3 3 +5 5 +7 7 +9 9 + +-- !hms_true_pruning -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !hms_true_count -- +21 + +-- !hms_true_dv_splits -- +1 merged-1 1 +10 value-10 0 +11 updated-11 1 +12 value-12 0 +13 value-13 1 +15 value-15 1 +16 value-16 0 +17 value-17 1 +18 value-18 0 +19 value-19 1 +2 value-2 0 +20 value-20 0 +22 updated-22 0 +23 value-23 1 +24 value-24 0 +25 value-25 1 +26 value-26 0 +27 value-27 1 +29 value-29 1 +3 value-3 1 +30 value-30 0 +31 value-31 1 +32 value-32 0 +33 updated-33 1 +34 value-34 0 +36 value-36 0 +37 value-37 1 +38 value-38 0 +39 value-39 1 +4 value-4 0 +40 value-40 0 +41 value-41 1 +43 value-43 1 +44 updated-44 0 +45 value-45 1 +46 value-46 0 +47 value-47 1 +48 value-48 0 +5 value-5 1 +50 value-50 0 +51 value-51 1 +52 value-52 0 +53 value-53 1 +54 value-54 0 +55 updated-55 1 +57 value-57 1 +58 value-58 0 +59 value-59 1 +6 value-6 0 +60 value-60 0 +61 value-61 1 +62 value-62 0 +64 value-64 0 +65 value-65 1 +66 updated-66 0 +67 value-67 1 +68 value-68 0 +69 value-69 1 +71 value-71 1 +72 value-72 0 +73 value-73 1 +74 value-74 0 +75 value-75 1 +76 value-76 0 +78 value-78 0 +79 value-79 1 +8 value-8 0 +80 value-80 0 +81 value-81 1 +82 value-82 0 +83 value-83 1 +85 value-85 1 +86 value-86 0 +87 value-87 1 +88 updated-88 0 +89 value-89 1 +9 value-9 1 +90 value-90 0 +92 value-92 0 +93 value-93 1 +94 value-94 0 +95 value-95 1 +96 value-96 0 +97 value-97 1 +99 updated-99 1 + +-- !hms_true_mapping_splits -- +1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_initial -- +1 + +-- !hms_cached -- +1 + +-- !hms_uncached -- +1 +2 + +-- !hms_table_refresh -- +1 +2 + +-- !hms_cached_schema -- +id bigint Yes false \N + +-- !hms_refreshed_schema -- +id bigint Yes false \N +value text Yes false \N + +-- !hms_schema_rows -- +1 \N +2 \N +3 new-schema + +-- !hms_database_refresh -- +1 \N +2 \N +3 new-schema +4 database-refresh + +-- !hms_catalog_refresh -- +1 \N +2 \N +3 new-schema +4 database-refresh +5 catalog-refresh + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_metrics.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_metrics.out new file mode 100644 index 00000000000000..a186ecf5268dcc --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_metrics.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filter_false -- +0 low-0 +1 low-1 +2 low-2 +3 low-3 +4 low-4 + +-- !count_false -- +21 + +-- !partition_false -- +1 + +-- !filter_true -- +0 low-0 +1 low-1 +2 low-2 +3 low-3 +4 low-4 + +-- !count_true -- +21 + +-- !partition_true -- +1 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet.out new file mode 100644 index 00000000000000..200c15ff7c5c40 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet.out @@ -0,0 +1,133 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_none -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + +-- !filesystem_name -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + +-- !filesystem_id -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + +-- !hms_none -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + +-- !hms_name -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + +-- !hms_id -- +100 64 +101 64 +102 64 +103 64 +104 64 +105 64 +106 64 +107 64 +108 64 +109 64 +110 64 +111 64 +112 64 +113 64 +114 64 +115 64 +116 64 +117 64 +118 64 +119 64 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.out new file mode 100644 index 00000000000000..003e724f84140a --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.out @@ -0,0 +1,259 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_early -- +100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +107 row-107xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +108 row-108xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +109 row-109xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +110 row-110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +111 row-111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +112 row-112xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +113 row-113xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +114 row-114xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +115 row-115xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +116 row-116xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +117 row-117xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +118 row-118xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_early_no_pruning -- +100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +107 row-107xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +108 row-108xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +109 row-109xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +110 row-110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +111 row-111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +112 row-112xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +113 row-113xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +114 row-114xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +115 row-115xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +116 row-116xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +117 row-117xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +118 row-118xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_early_count -- +17 + +-- !filesystem_page_edge -- +195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +202 row-202xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +203 row-203xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_page_edge_no_pruning -- +195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +202 row-202xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +203 row-203xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_page_edge_count -- +8 + +-- !filesystem_late -- +9000 row-9000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9001 row-9001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9002 row-9002xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9003 row-9003xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9004 row-9004xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9007 row-9007xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9008 row-9008xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9009 row-9009xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9010 row-9010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9011 row-9011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9012 row-9012xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9013 row-9013xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9014 row-9014xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9015 row-9015xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9016 row-9016xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9017 row-9017xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9018 row-9018xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_late_no_pruning -- +9000 row-9000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9001 row-9001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9002 row-9002xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9003 row-9003xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9004 row-9004xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9007 row-9007xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9008 row-9008xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9009 row-9009xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9010 row-9010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9011 row-9011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9012 row-9012xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9013 row-9013xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9014 row-9014xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9015 row-9015xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9016 row-9016xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9017 row-9017xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9018 row-9018xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_late_count -- +18 + +-- !filesystem_disjoint_pages -- +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_all_deleted -- + +-- !filesystem_all_deleted_aggregates -- +0 \N \N + +-- !filesystem_cross_split -- +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_count -- +9990 + +-- !filesystem_min_max -- +1 9998 + +-- !filesystem_misaligned_payloads -- +0 + +-- !hms_early -- +100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +107 row-107xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +108 row-108xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +109 row-109xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +110 row-110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +111 row-111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +112 row-112xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +113 row-113xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +114 row-114xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +115 row-115xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +116 row-116xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +117 row-117xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +118 row-118xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_early_no_pruning -- +100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +107 row-107xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +108 row-108xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +109 row-109xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +110 row-110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +111 row-111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +112 row-112xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +113 row-113xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +114 row-114xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +115 row-115xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +116 row-116xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +117 row-117xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +118 row-118xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_early_count -- +17 + +-- !hms_page_edge -- +195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +202 row-202xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +203 row-203xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_page_edge_no_pruning -- +195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +202 row-202xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +203 row-203xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_page_edge_count -- +8 + +-- !hms_late -- +9000 row-9000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9001 row-9001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9002 row-9002xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9003 row-9003xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9004 row-9004xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9007 row-9007xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9008 row-9008xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9009 row-9009xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9010 row-9010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9011 row-9011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9012 row-9012xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9013 row-9013xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9014 row-9014xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9015 row-9015xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9016 row-9016xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9017 row-9017xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9018 row-9018xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_late_no_pruning -- +9000 row-9000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9001 row-9001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9002 row-9002xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9003 row-9003xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9004 row-9004xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9007 row-9007xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9008 row-9008xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9009 row-9009xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9010 row-9010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9011 row-9011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9012 row-9012xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9013 row-9013xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9014 row-9014xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9015 row-9015xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9016 row-9016xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9017 row-9017xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9018 row-9018xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_late_count -- +18 + +-- !hms_disjoint_pages -- +201 row-201xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +204 row-204xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_all_deleted -- + +-- !hms_all_deleted_aggregates -- +0 \N \N + +-- !hms_cross_split -- +102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +9006 row-9006xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_count -- +9990 + +-- !hms_min_max -- +1 9998 + +-- !hms_misaligned_payloads -- +0 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_scan_reuse.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_scan_reuse.out new file mode 100644 index 00000000000000..b2c7a2e2098725 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_scan_reuse.out @@ -0,0 +1,244 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_off_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_off_count -- +filtered 5 +first 9990 +second 9990 + +-- !filesystem_off_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_on_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_on_count -- +filtered 5 +first 9990 +second 9990 + +-- !filesystem_on_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !filesystem_stream_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !filesystem_stream_count -- +filtered 5 +first 9990 +second 9990 + +-- !filesystem_stream_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_off_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_off_count -- +filtered 5 +first 9990 +second 9990 + +-- !hms_off_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_on_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_on_count -- +filtered 5 +first 9990 +second 9990 + +-- !hms_on_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !hms_stream_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !hms_stream_count -- +filtered 5 +first 9990 +second 9990 + +-- !hms_stream_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !gateway_off_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !gateway_off_count -- +filtered 5 +first 9990 +second 9990 + +-- !gateway_off_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !gateway_on_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !gateway_on_count -- +filtered 5 +first 9990 +second 9990 + +-- !gateway_on_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + +-- !gateway_stream_ranges -- +different 195 row-195xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 196 row-196xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 197 row-197xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +different 198 row-198xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +first 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 100 row-100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 102 row-102xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 103 row-103xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 104 row-104xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +second 106 row-106xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +-- !gateway_stream_count -- +filtered 5 +first 9990 +second 9990 + +-- !gateway_stream_mapping -- +first 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +first 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value +second 1 {"after":10, "keep":null} [{"after":100}] {"k":{"after":1000}} a/b \N +second 2 {"after":20, "keep":"new-nested"} [{"after":200}] {"k":{"after":2000}} \N new-value + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_skipping.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_skipping.out new file mode 100644 index 00000000000000..26dc697cf8968a --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_skipping.out @@ -0,0 +1,178 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all -- +\N \N \N \N \N +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +100 high-100 100.0000 100.0 100 +101 high-101 101.0000 101.0 101 +102 high-102 102.0000 102.0 102 +103 high-103 103.0000 103.0 103 +104 high-104 104.0000 104.0 104 +105 high-105 105.0000 105.0 105 +106 high-106 106.0000 106.0 106 +107 high-107 107.0000 107.0 107 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 +8 low-8 8.0000 8.0 8 +9 low-9 9.0000 9.0 9 + +-- !range -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !reversed -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !between -- +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 + +-- !or -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 + +-- !null -- +\N \N \N \N \N + +-- !not_null -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +100 high-100 100.0000 100.0 100 +101 high-101 101.0000 101.0 101 +102 high-102 102.0000 102.0 102 +103 high-103 103.0000 103.0 103 +104 high-104 104.0000 104.0 104 +105 high-105 105.0000 105.0 105 +106 high-106 106.0000 106.0 106 +107 high-107 107.0000 107.0 107 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 +8 low-8 8.0000 8.0 8 +9 low-9 9.0000 9.0 9 + +-- !not_null_safe -- +\N \N \N \N \N +0 low-0 0.0000 0.0 0 +100 high-100 100.0000 100.0 100 +101 high-101 101.0000 101.0 101 +102 high-102 102.0000 102.0 102 +103 high-103 103.0000 103.0 103 +104 high-104 104.0000 104.0 104 +105 high-105 105.0000 105.0 105 +106 high-106 106.0000 106.0 106 +107 high-107 107.0000 107.0 107 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 +8 low-8 8.0000 8.0 8 +9 low-9 9.0000 9.0 9 + +-- !in -- +1 low-1 1.0000 1.0 1 +101 high-101 101.0000 101.0 101 + +-- !not_in -- + +-- !partial_and -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 + +-- !partial_or -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 +8 low-8 8.0000 8.0 8 +9 low-9 9.0000 9.0 9 + +-- !partial_not -- +0 low-0 0.0000 0.0 0 +1 low-1 1.0000 1.0 1 +100 high-100 100.0000 100.0 100 +101 high-101 101.0000 101.0 101 +102 high-102 102.0000 102.0 102 +103 high-103 103.0000 103.0 103 +104 high-104 104.0000 104.0 104 +105 high-105 105.0000 105.0 105 +106 high-106 106.0000 106.0 106 +107 high-107 107.0000 107.0 107 +2 low-2 2.0000 2.0 2 +3 low-3 3.0000 3.0 3 +4 low-4 4.0000 4.0 4 +5 low-5 5.0000 5.0 5 +6 low-6 6.0000 6.0 6 +7 low-7 7.0000 7.0 7 +8 low-8 8.0000 8.0 8 +9 low-9 9.0000 9.0 9 + +-- !decimal -- +100 high-100 100.0000 100.0 100 +101 high-101 101.0000 101.0 101 +102 high-102 102.0000 102.0 102 +103 high-103 103.0000 103.0 103 +104 high-104 104.0000 104.0 104 +105 high-105 105.0000 105.0 105 +106 high-106 106.0000 106.0 106 +107 high-107 107.0000 107.0 107 +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 + +-- !float -- +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 + +-- !double -- +108 high-108 108.0000 108.0 108 +109 high-109 109.0000 109.0 109 + +-- !count -- +21 + +-- !count_column -- +20 + +-- !filtered_count -- +5 + +-- !function_count -- +10 + +-- !count_without_pushdown -- +21 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_statistics.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_statistics.out new file mode 100644 index 00000000000000..e0c549184f4cb8 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_statistics.out @@ -0,0 +1,64 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_delta_m1_basic_count -- +3 + +-- !filesystem_delta_golden_dv_partitioned_with_checkpoint_count -- +35 + +-- !filesystem_delta_golden_v2_checkpoint_json_count -- +10 + +-- !filesystem_delta_golden_v2_checkpoint_parquet_count -- +10 + +-- !filesystem_refreshed_count -- +3 + +-- !filesystem_empty_count -- +0 + +-- !filesystem_disabled_count -- +35 + +-- !hms_delta_m1_basic_count -- +3 + +-- !hms_delta_golden_dv_partitioned_with_checkpoint_count -- +35 + +-- !hms_delta_golden_v2_checkpoint_json_count -- +10 + +-- !hms_delta_golden_v2_checkpoint_parquet_count -- +10 + +-- !hms_refreshed_count -- +3 + +-- !hms_empty_count -- +0 + +-- !hms_disabled_count -- +35 + +-- !hive_delta_m1_basic_count -- +3 + +-- !hive_delta_golden_dv_partitioned_with_checkpoint_count -- +35 + +-- !hive_delta_golden_v2_checkpoint_json_count -- +10 + +-- !hive_delta_golden_v2_checkpoint_parquet_count -- +10 + +-- !hive_refreshed_count -- +3 + +-- !hive_empty_count -- +0 + +-- !hive_disabled_count -- +35 + diff --git a/regression-test/data/external_table_p0/deltalake/test_deltalake_version_as_of.out b/regression-test/data/external_table_p0/deltalake/test_deltalake_version_as_of.out new file mode 100644 index 00000000000000..eef0df6463df08 --- /dev/null +++ b/regression-test/data/external_table_p0/deltalake/test_deltalake_version_as_of.out @@ -0,0 +1,91 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filesystem_warm_latest -- +20 + +-- !hms_warm_latest -- +20 + +-- !gateway_warm_latest -- +20 + +-- !filesystem_id_original -- +1 old-value + +-- !hms_id_original -- +1 old-value + +-- !gateway_id_original -- +1 old-value + +-- !filesystem_id_dropped -- +1 + +-- !hms_id_dropped -- +1 + +-- !gateway_id_dropped -- +1 + +-- !filesystem_id_readded -- +1 \N +2 10000000000 + +-- !hms_id_readded -- +1 \N +2 10000000000 + +-- !gateway_id_readded -- +1 \N +2 10000000000 + +-- !filesystem_dropped_partition -- +0 row-0 + +-- !hms_dropped_partition -- +0 row-0 + +-- !gateway_dropped_partition -- +0 row-0 + +-- !filesystem_dv_counts -- +after 18 +before 20 +masked 0 + +-- !hms_dv_counts -- +after 18 +before 20 +masked 0 + +-- !gateway_dv_counts -- +after 18 +before 20 +masked 0 + +-- !filesystem_cached_latest -- +20 + +-- !filesystem_mixed_latest -- +row-2 \N + +-- !filesystem_refreshed_latest -- +19 + +-- !hms_cached_latest -- +20 + +-- !hms_mixed_latest -- +row-2 \N + +-- !hms_refreshed_latest -- +19 + +-- !gateway_cached_latest -- +20 + +-- !gateway_mixed_latest -- +row-2 \N + +-- !gateway_refreshed_latest -- +19 + diff --git a/regression-test/data/external_table_p2/deltalake/test_deltalake_planning.out b/regression-test/data/external_table_p2/deltalake/test_deltalake_planning.out new file mode 100644 index 00000000000000..d322a826455286 --- /dev/null +++ b/regression-test/data/external_table_p2/deltalake/test_deltalake_planning.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cold_count -- +1000000 + +-- !warm_count -- +1000000 + diff --git a/regression-test/data/external_table_p2/deltalake/test_deltalake_storage.out b/regression-test/data/external_table_p2/deltalake/test_deltalake_storage.out new file mode 100644 index 00000000000000..ca8632d419ca3e --- /dev/null +++ b/regression-test/data/external_table_p2/deltalake/test_deltalake_storage.out @@ -0,0 +1,249 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !false_mapping -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !false_dv -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !false_dv_count -- +35 + +-- !false_partitioned_dv -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !false_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !false_widening -- +{"a":1} {2:3} [4, 5] +{"a":9223372036854775807} {9223372036854775807:9223372036854775807} [9223372036854775807, -9223372036854775808] + +-- !false_checkpoint -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !false_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !true_mapping -- +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N +0 0 0 0 0.0 0 0.00 true 0 0x30 2021-11-18 1970-01-01 00:00:00.000000+00:00 {"aa":"0", "ac":{"aca":0}} [0, 1] [[0, 1], [2, 3]] [{0:[2, 3], 1:[4, 5]}] [{"ab":0}, {"ab":0}] {"aa":[0, 1], "ab":{[0, 1]:{"aca":2}}} {0:1, 2:3} {1:{"ab":0}} {0:[0, null, 1], 1:[]} {0:{0:0}, 1:{2:0}} +1 1 1 1 1.0 1 1.00 false 1 0x31 2021-11-18 1970-01-01 00:00:00.001000+00:00 {"aa":"1", "ac":{"aca":1}} [1, 2] [[1, 2], [3, 4]] [{1:[2, 3], 2:[4, 5]}] [{"ab":1}, {"ab":1}] {"aa":[1, 2], "ab":{[1, 2]:{"aca":3}}} {1:2, 3:4} {2:{"ab":20}} {1:[1, null, 2], 2:[]} {1:{1:1}, 2:{3:1}} +2 2 2 2 2.0 2 2.00 true 2 0x32 2021-11-18 1970-01-01 00:00:00.002000+00:00 {"aa":"2", "ac":{"aca":2}} [2, 3] [[2, 3], [4, 5]] [{2:[2, 3], 3:[4, 5]}] [{"ab":2}, {"ab":2}] {"aa":[2, 3], "ab":{[2, 3]:{"aca":4}}} {2:3, 4:5} {3:{"ab":40}} {2:[2, null, 3], 3:[]} {2:{2:2}, 3:{4:2}} +3 3 3 3 3.0 3 3.00 false 3 0x33 2021-11-18 1970-01-01 00:00:00.003000+00:00 {"aa":"3", "ac":{"aca":3}} [3, 4] [[3, 4], [5, 6]] [{3:[2, 3], 4:[4, 5]}] [{"ab":3}, {"ab":3}] {"aa":[3, 4], "ab":{[3, 4]:{"aca":5}}} {3:4, 5:6} {4:{"ab":60}} {3:[3, null, 4], 4:[]} {3:{3:3}, 4:{5:3}} +4 4 4 4 4.0 4 4.00 true 4 0x34 2021-11-18 1970-01-01 00:00:00.004000+00:00 {"aa":"4", "ac":{"aca":4}} [4, 5] [[4, 5], [6, 7]] [{4:[2, 3], 5:[4, 5]}] [{"ab":4}, {"ab":4}] {"aa":[4, 5], "ab":{[4, 5]:{"aca":6}}} {4:5, 6:7} {5:{"ab":80}} {4:[4, null, 5], 5:[]} {4:{4:4}, 5:{6:4}} + +-- !true_dv -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !true_dv_count -- +35 + +-- !true_partitioned_dv -- +0 30 foo0 +0 40 foo0 +1 1 foo1 +1 11 foo1 +1 21 foo1 +1 31 foo1 +1 41 foo1 +2 32 foo2 +2 42 foo2 +3 13 foo3 +3 23 foo3 +3 3 foo3 +3 33 foo3 +3 43 foo3 +4 34 foo4 +4 44 foo4 +5 15 foo0 +5 25 foo0 +5 35 foo0 +5 45 foo0 +5 5 foo0 +6 36 foo1 +6 46 foo1 +7 17 foo2 +7 27 foo2 +7 37 foo2 +7 47 foo2 +7 7 foo2 +8 38 foo3 +8 48 foo3 +9 19 foo4 +9 29 foo4 +9 39 foo4 +9 49 foo4 +9 9 foo4 + +-- !true_ntz -- +0 2021-11-18T02:30:00.123456 2021-11-18T02:30:00.123456 +1 2013-07-05T17:01:00.123456 2021-11-18T02:30:00.123456 +2 \N 2021-11-18T02:30:00.123456 +3 2021-11-18T02:30:00.123456 2013-07-05T17:01:00.123456 +4 2013-07-05T17:01:00.123456 2013-07-05T17:01:00.123456 +5 \N 2013-07-05T17:01:00.123456 +6 2021-11-18T02:30:00.123456 \N +7 2013-07-05T17:01:00.123456 \N +8 \N \N + +-- !true_widening -- +{"a":1} {2:3} [4, 5] +{"a":9223372036854775807} {9223372036854775807:9223372036854775807} [9223372036854775807, -9223372036854775808] + +-- !true_checkpoint -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !true_checkpoint_json -- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_basic.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_basic.groovy new file mode 100644 index 00000000000000..9e5c5706bbf7cf --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_basic.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("test_deltalake_basic", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + def profiles = new ProfileAction(context) + sql "set time_zone='UTC'" + sql "set enable_profile=true" + sql "set enable_sql_cache=false" + for (String source : ["filesystem", "hms"]) { + String catalog = "test_deltalake_basic_${source}" + sql "drop catalog if exists ${catalog}" + String metastore = source == "filesystem" ? "'warehouse'='${warehouse}'" + : "'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + sql """create catalog ${catalog} properties ( + 'type'='deltalake', 'deltalake.catalog.type'='${source}', ${metastore}, + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "refresh catalog ${catalog}" + "order_qt_${source}_tables"("show tables from ${catalog}.delta_m1 like 'basic'") + "qt_${source}_schema"("desc ${catalog}.delta_m1.basic") + String v2Label = "${source}_true" + "order_qt_${v2Label}_basic"("select * from ${catalog}.delta_m1.basic order by id") + "qt_${v2Label}_count"("select count(*) from ${catalog}.delta_m1.basic") + "order_qt_${v2Label}_null_partition"( + "select * from ${catalog}.delta_m1.basic where part is null order by id") + String token = UUID.randomUUID().toString() + "order_qt_${v2Label}_one_partition"( + "select /* ${token} */ * from ${catalog}.delta_m1.basic where part='a' order by id") + String profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + def metrics = profile.readLines()*.trim() + assertTrue(metrics.contains("- SelectedFiles: 1"), profile) + assertTrue(metrics.contains("- GeneratedSplits: 1"), profile) + "order_qt_${v2Label}_no_partition"( + "select * from ${catalog}.delta_m1.basic where part='missing' order by id") + "order_qt_${v2Label}_empty"("select * from ${catalog}.delta_m1.empty order by id") + "qt_${v2Label}_empty_count"("select count(*) from ${catalog}.delta_m1.empty") + "order_qt_${v2Label}_types"("""select id,b,i8,i16,i32,i64,f,d,dec,str,hex(bin),dt,ts,arr,m,nested + from ${catalog}.delta_m1.types order by id""") + "order_qt_${v2Label}_partition_types"( + "select * from ${catalog}.delta_m1.partition_types order by id") + explain { + sql("select * from ${catalog}.delta_m1.basic where part='a'") + contains("partition=1/3") + } + sql "set file_split_size=128" + "order_qt_${source}_byte_ranges"("""select id,b,i8,i16,i32,i64,f,d,dec,str,hex(bin),dt,ts,arr,m,nested + from ${catalog}.delta_m1.types order by id""") + sql "set file_split_size=0" + for (String zone : ["UTC", "Asia/Shanghai", "America/Los_Angeles"]) { + sql "set time_zone='${zone}'" + String zoneLabel = zone.replace('/', '_') + "order_qt_${source}_${zoneLabel}_timestamps"( + "select id,cast(ts as datetimev2(6)) from ${catalog}.delta_m1.types order by id") + "order_qt_${source}_${zoneLabel}_partition_timestamps"( + "select id,cast(p_ts as datetimev2(6)) from ${catalog}.delta_m1.partition_types order by id") + } + sql "set time_zone='UTC'" + test { + sql "select * from ${catalog}.delta_m1.missing_table" + exception "does not exist" + } + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_cast_pruning.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_cast_pruning.groovy new file mode 100644 index 00000000000000..9338ff688d1439 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_cast_pruning.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("test_deltalake_cast_pruning", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "set enable_sql_cache=false" + sql "set enable_profile=true" + sql "set num_files_in_batch_mode=1" + def profiles = new ProfileAction(context) + + for (String source : ["filesystem", "hms", "gateway"]) { + String catalog = "test_deltalake_cast_pruning_${source}" + String connector = source == "gateway" ? "'type'='hms'" + : "'type'='deltalake', 'deltalake.catalog.type'='${source}'" + String location = source == "filesystem" ? "'warehouse'='${warehouse}'" + : "'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + ${connector}, ${location}, 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "switch ${catalog}" + sql "use delta_golden" + + // The golden has two files. The first one's decimal min and max are both 123.45. + String column = "decimal_decimal_same_scale" + String castMatch = "cast(${column} as bigint) = 123" + for (boolean batch : [false, true]) { + sql "set enable_external_table_batch_mode=${batch}" + String label = "${source}_${batch ? 'stream' : 'sync'}" + "order_qt_${label}_cast"( + "select ${column}, cast(${column} as bigint) from type_widening where ${castMatch}") + // Only the second file matches. A table-level count or a first-file-only fallback would be wrong. + "qt_${label}_count"( + "select count(*) from type_widening where cast(${column} as bigint) = 12345678901234") + "order_qt_${label}_try_or"("""select ${column} from type_widening + where try_cast(${column} as bigint) = 123 or ${column} < 0""") + String mixed = "${castMatch} and ${column} < 124" + String token = UUID.randomUUID().toString() + "order_qt_${label}_and"("select /* ${token} */ ${column} from type_widening where ${mixed}") + // The independent decimal condition still prunes the second file in both execution modes. + String profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + assertTrue(profile.contains("TotalFiles: 2") && profile.contains("SelectedFiles: 1"), profile) + } + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_errors.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_errors.groovy new file mode 100644 index 00000000000000..94797a1968b240 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_errors.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_deltalake_errors", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "drop catalog if exists test_deltalake_errors" + sql """create catalog test_deltalake_errors properties ( + 'type'='deltalake', 'warehouse'='${warehouse}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + test { + sql "select * from test_deltalake_errors.delta_golden.spark_variant_checkpoint" + exception "Unsupported Delta Lake reader features [variantType-preview]" + } + test { + sql "select * from test_deltalake_errors.delta_golden.collations_table" + exception "Unsupported Delta Lake type: string collate UTF8_LCASE" + } + test { + sql "select * from test_deltalake_errors.delta_golden.no_delta_log_folder" + exception "Table [no_delta_log_folder] does not exist" + } + test { + sql "select * from test_deltalake_errors.delta_golden.does_not_exist" + exception "Table [does_not_exist] does not exist" + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_features.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_features.groovy new file mode 100644 index 00000000000000..7b63ff8410330d --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_features.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_deltalake_features", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "set time_zone='UTC'" + for (String source : ["filesystem", "hms"]) { + String catalog = "test_deltalake_features_${source}" + sql "drop catalog if exists ${catalog}" + String metastore = source == "filesystem" ? "'warehouse'='${warehouse}'" + : "'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + sql """create catalog ${catalog} properties ( + 'type'='deltalake', 'deltalake.catalog.type'='${source}', ${metastore}, + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + for (String mode : ["name", "id"]) { + String table = "${catalog}.delta_m1.mapping_${mode}" + "qt_${source}_${mode}_schema"("desc ${table}") + "order_qt_${source}_${mode}_all"("select * from ${table}") + "order_qt_${source}_${mode}_filter"("select * from ${table} where renamed=1") + "order_qt_${source}_${mode}_partition"("select * from ${table} where partition_renamed='a/b'") + "order_qt_${source}_${mode}_null_partition"("select * from ${table} where partition_renamed is null") + "order_qt_${source}_${mode}_nested"("""select renamed, + struct_element(nested, 'after'), struct_element(nested, 'keep'), + struct_element(arr[1], 'after'), struct_element(m['k'], 'after') from ${table} + where struct_element(nested, 'after')=10""") + "order_qt_${source}_${mode}_golden"( + "select * from ${catalog}.delta_golden.table_with_columnmapping_mode_${mode}") + } + for (String zone : ["UTC", "Asia/Shanghai", "America/Los_Angeles"]) { + sql "set time_zone='${zone}'" + String label = "${source}_${zone.replace('/', '_')}" + "order_qt_${label}_ntz"("select * from ${catalog}.delta_m1.ntz") + "order_qt_${label}_ntz_filter"("""select * from ${catalog}.delta_m1.ntz + where ts='2024-03-10 02:30:00.123456' and part='2024-03-10 02:30:00.123456'""") + for (String suffix : ["", "_name_mode", "_id_mode"]) { + "order_qt_${label}_golden_ntz${suffix}"( + "select * from ${catalog}.delta_golden.data_reader_timestamp_ntz${suffix}") + } + } + sql "set time_zone='UTC'" + "order_qt_${source}_widening"("select * from ${catalog}.delta_m1.widening") + "order_qt_${source}_widening_filter"("""select * from ${catalog}.delta_m1.widening + where small > 32767 and int_decimal < 200 and dt > '2024-02-29 00:00:00'""") + "order_qt_${source}_golden_widening"("select * from ${catalog}.delta_golden.type_widening") + "order_qt_${source}_golden_nested_widening"("select * from ${catalog}.delta_golden.type_widening_nested") + for (boolean countPushdown : [true, false]) { + sql "set enable_count_push_down_for_external_table=${countPushdown}" + String label = "${source}_${countPushdown}" + "qt_${label}_dv_count"("select count(*) from ${catalog}.delta_m1.dv") + "qt_${label}_dv_column_count"("select count(value) from ${catalog}.delta_m1.dv") + "qt_${label}_dv_filtered_count"("select count(*) from ${catalog}.delta_m1.dv where part=1 and id<20") + for (String table : ["dv_partitioned_with_checkpoint", "dv_with_columnmapping", "log_replay_dv_key_cases"]) { + "qt_${label}_${table}_count"("select count(*) from ${catalog}.delta_golden.${table}") + } + } + String v2Label = "${source}_true" + "order_qt_${v2Label}_dv_rows"("select * from ${catalog}.delta_m1.dv") + "order_qt_${v2Label}_dv_deleted"("select * from ${catalog}.delta_m1.dv where id%7=0") + "order_qt_${v2Label}_dv_partition"("select * from ${catalog}.delta_m1.dv where part=1 and id<20") + "order_qt_${v2Label}_dv_updates"("select * from ${catalog}.delta_m1.dv where value like 'updated-%'") + for (String table : ["dv_partitioned_with_checkpoint", "dv_with_columnmapping", "log_replay_dv_key_cases"]) { + "order_qt_${v2Label}_${table}"("select * from ${catalog}.delta_golden.${table}") + } + // Exercise file-global DV positions when a Parquet file spans many scan ranges. + sql "set file_split_size=128" + "order_qt_${source}_dv_byte_ranges"("select * from ${catalog}.delta_m1.dv") + "order_qt_${source}_mapping_byte_ranges"("select * from ${catalog}.delta_m1.mapping_name") + "order_qt_${source}_golden_dv_byte_ranges"("select * from ${catalog}.delta_golden.dv_with_columnmapping") + sql "set file_split_size=0" + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_hive.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hive.groovy new file mode 100644 index 00000000000000..82251e355c11ab --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hive.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.JdbcUtils + +import java.sql.DriverManager + +suite("test_deltalake_hive", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String sparkPort = context.config.otherConfigs.get("deltalakeSparkPort") + sql "drop catalog if exists test_deltalake_hive" + sql """create catalog test_deltalake_hive properties ( + 'type'='hms', 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 'meta.cache.deltalake.latest.ttl-second'='-1', + 'deltalake.catalog.type'='filesystem', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "switch test_deltalake_hive" + sql "use delta_m1" + sql "set time_zone='UTC'" + qt_mapping_schema "desc mapping_id" + qt_mapping_ddl "show create table mapping_id" + for (String table : ["mapping_name", "mapping_id", "ntz", "widening", "dv", "log_compaction"]) { + "order_qt_${table}"("select * from ${table}") + "qt_${table}_count"("select count(*) from ${table}") + } + order_qt_mapping_filter "select * from mapping_name where partition_renamed='a/b'" + order_qt_dv_filter "select * from dv where part=1 and id<20" + order_qt_v2_json "select * from delta_golden.v2_checkpoint_json" + order_qt_v2_parquet "select * from delta_golden.v2_checkpoint_parquet" + sql "set enable_external_table_batch_mode=true" + sql "set num_files_in_batch_mode=1" + sql "set file_split_size=128" + order_qt_dv_streaming "select * from dv" + order_qt_mapping_streaming "select * from mapping_id" + sql "set file_split_size=0" + Class.forName("org.apache.hive.jdbc.HiveDriver") + def spark = DriverManager.getConnection("jdbc:hive2://${host}:${sparkPort}/default", "hadoop", "hadoop") + try { + JdbcUtils.executeToList(spark, """create or replace table delta_m1.gateway_refresh (id bigint) + using delta tblproperties ('delta.enableDeletionVectors'='false')""") + JdbcUtils.executeToList(spark, "insert into delta_m1.gateway_refresh values (1)") + // The gateway was resolved before Spark registered this table. + sql "refresh database delta_m1" + order_qt_initial "select * from gateway_refresh" + JdbcUtils.executeToList(spark, "insert into delta_m1.gateway_refresh values (2)") + order_qt_cached "select * from gateway_refresh" + sql "refresh table gateway_refresh" + order_qt_table_refresh "select * from gateway_refresh" + JdbcUtils.executeToList(spark, "alter table delta_m1.gateway_refresh add columns (value string)") + JdbcUtils.executeToList(spark, "insert into delta_m1.gateway_refresh values (3, 'catalog-refresh')") + sql "refresh catalog test_deltalake_hive" + qt_refreshed_schema "desc gateway_refresh" + order_qt_schema_refresh "select * from gateway_refresh" + JdbcUtils.executeToList(spark, "insert into delta_m1.gateway_refresh values (4, 'database-refresh')") + sql "refresh database delta_m1" + order_qt_database_refresh "select * from gateway_refresh" + JdbcUtils.executeToList(spark, "drop table if exists delta_m1.gateway_plain") + JdbcUtils.executeToList(spark, """create table delta_m1.gateway_plain (id bigint, value string) + using parquet location 's3a://delta/warehouse/delta_m1/gateway_plain'""") + JdbcUtils.executeToList(spark, "insert overwrite delta_m1.gateway_plain values (1, 'hive'), (2, 'parquet')") + sql "refresh database delta_m1" + order_qt_plain_hive "select * from gateway_plain" + } finally { + spark.close() + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.groovy new file mode 100644 index 00000000000000..09a41e4935c106 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_mixed_cast.groovy @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +// Mixed-format reads through one HMS catalog must preserve each table's matching rows. +suite("test_deltalake_hms_mixed_cast", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + sql "set enable_sql_cache=false" + sql "set enable_profile=true" + sql "set num_files_in_batch_mode=1" + def profiles = new ProfileAction(context) + + for (String kind : ["iceberg", "hms"]) { + String catalog = "test_deltalake_hms_mixed_cast_${kind}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + 'type'='${kind}', 'iceberg.catalog.type'='hms', + 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + } + + // Reuse the retained Delta golden. Create two Iceberg files in that same HMS database. + // Integral decimal values match with or without Iceberg CAST pushdown. + sql "switch test_deltalake_hms_mixed_cast_iceberg" + sql "use delta_golden" + sql "drop table if exists test_deltalake_hms_mixed_cast_iceberg" + sql """create table test_deltalake_hms_mixed_cast_iceberg (id int, amount decimal(18, 2)) + properties ('format-version'='2')""" + sql "insert into test_deltalake_hms_mixed_cast_iceberg values (123, 123.00)" + sql "insert into test_deltalake_hms_mixed_cast_iceberg values (999, 999.00)" + order_qt_iceberg_fixture "select id, amount from test_deltalake_hms_mixed_cast_iceberg" + qt_iceberg_files "select count(*) from test_deltalake_hms_mixed_cast_iceberg\$files where content = 0" + + sql "switch test_deltalake_hms_mixed_cast_hms" + sql "use delta_golden" + String deltaCast = "cast(decimal_decimal_same_scale as bigint) = 123" + String icebergSelect = """select 'iceberg' as source, amount + from test_deltalake_hms_mixed_cast_iceberg where cast(amount as bigint) = 123""" + String deltaSelect = """select 'delta' as source, decimal_decimal_same_scale as amount + from type_widening where ${deltaCast}""" + + for (boolean batch : [false, true]) { + sql "set enable_external_table_batch_mode=${batch}" + String mode = batch ? "stream" : "sync" + for (String variant : ["delta_first", "iceberg_first", "plain_and"]) { + String deltaQuery = variant == "plain_and" + ? "${deltaSelect} and decimal_decimal_same_scale < 124" : deltaSelect + String mixed = variant == "iceberg_first" ? "${icebergSelect} union all ${deltaQuery}" + : "${deltaQuery} union all ${icebergSelect}" + // Check Iceberg's results without pinning its CAST policy, which may change to fix lossy casts. + String token = UUID.randomUUID().toString() + "order_qt_${mode}_${variant}"("select /* ${token} */ * from (${mixed}) mixed") + String profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + int selected = variant == "plain_and" ? 1 : 2 + assertTrue(profile.contains("TotalFiles: 2") && profile.contains("SelectedFiles: ${selected}"), profile) + } + + // Both scans in a real join also share the HMS catalog's statement metadata. + "order_qt_${mode}_join_delta_first"("""select d.decimal_decimal_same_scale, i.id, i.amount + from type_widening d join test_deltalake_hms_mixed_cast_iceberg i + on cast(d.decimal_decimal_same_scale as bigint) = i.id + where cast(d.decimal_decimal_same_scale as bigint) = 123 and cast(i.amount as bigint) = 123""") + "order_qt_${mode}_join_iceberg_first"("""select d.decimal_decimal_same_scale, i.id, i.amount + from test_deltalake_hms_mixed_cast_iceberg i join type_widening d + on i.id = cast(d.decimal_decimal_same_scale as bigint) + where cast(i.amount as bigint) = 123 and cast(d.decimal_decimal_same_scale as bigint) = 123""") + "order_qt_${mode}_iceberg_after"(icebergSelect) + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_registration.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_registration.groovy new file mode 100644 index 00000000000000..96402ea0bb46b4 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_hms_registration.groovy @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.JdbcUtils + +import java.sql.DriverManager + +// HMS gives a Delta table only three things: whether it is Delta, its name and its root path. Schema, files and +// deletion vectors always come from _delta_log, and other suites cover those. This suite varies how Spark registers +// a table, so a path derived from the table name, or a stale name or path after a change, fails here. +// +// Every scenario runs through both catalogs that can reach such a table: the dedicated Delta catalog and the HMS +// gateway catalog, which serves plain Hive tables next to Delta ones. test_iceberg_on_hms_gateway_ddl_parity and +// test_iceberg_hms_case_sensibility are the models for that parity and for the name-case modes. +suite("test_deltalake_hms_registration", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String sparkPort = context.config.otherConfigs.get("deltalakeSparkPort") + // External tables keep their files after DROP TABLE, so each run needs a root no earlier run has written to. + String root = "s3a://delta/hms_registration/${System.currentTimeMillis()}" + Map catalogTypes = [ + "dedicated": "'type'='deltalake', 'deltalake.catalog.type'='hms'", + "gateway": "'type'='hms'", + ] + + def createCatalog = { String catalog, String typeProperties, String extraProperties -> + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + ${typeProperties}, 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 'meta.cache.deltalake.latest.ttl-second'='-1', ${extraProperties} + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + } + + Class.forName("org.apache.hive.jdbc.HiveDriver") + def spark = DriverManager.getConnection("jdbc:hive2://${host}:${sparkPort}/default", "hadoop", "hadoop") + def runSpark = { String statement -> JdbcUtils.executeToList(spark, statement) } + try { + for (String database : ["delta_hms_reg", "delta_hms_reg_plain", "delta_hms_reg_dropped"]) { + runSpark("drop database if exists ${database} cascade") + runSpark("create database ${database} location '${root}/${database}'") + } + + // Managed: no LOCATION, Spark chooses the path. Partitions and a deletion vector prove that data files and + // the deletion vector both resolve against the path HMS returned. + runSpark("""create table delta_hms_reg.reg_managed (id bigint, value string, part string) using delta + partitioned by (part) tblproperties ('delta.enableDeletionVectors'='true')""") + runSpark("""insert into delta_hms_reg.reg_managed values + (1, 'one', 'a'), (2, 'two', 'a'), (3, 'three', 'b'), (4, 'four', null)""") + runSpark("delete from delta_hms_reg.reg_managed where id = 2") + runSpark("""create table delta_hms_reg.reg_ctas using delta + as select 1L as id, 'ctas-one' as value union all select 2L, 'ctas-two'""") + + // External: the directory name has nothing to do with the table name. + runSpark("""create table delta_hms_reg.reg_external (id bigint, value string) using delta + location '${root}/elsewhere/not_the_table_name'""") + runSpark("insert into delta_hms_reg.reg_external values (1, 'external-one'), (2, 'external-two')") + // No columns in the DDL, and a second name for the same path: only _delta_log can supply this schema. + runSpark("""create table delta_hms_reg.reg_no_columns using delta + location '${root}/elsewhere/not_the_table_name'""") + runSpark("""create table delta_hms_reg.reg_space_path (id bigint) using delta + location '${root}/elsewhere/with space/table'""") + runSpark("insert into delta_hms_reg.reg_space_path values (1)") + runSpark("""create table delta_hms_reg.reg_trailing_slash (id bigint) using delta + location '${root}/elsewhere/trailing_slash/'""") + runSpark("insert into delta_hms_reg.reg_trailing_slash values (1)") + // HMS lowercases this name, while _delta_log keeps the case of the columns. + runSpark("create table delta_hms_reg.REG_Mixed_Case (ID bigint, MixedValue string) using delta") + runSpark("insert into delta_hms_reg.reg_mixed_case values (1, 'mixed')") + + // These share a database with Delta tables, and must be filtered or served as non-Delta. + runSpark("""create table delta_hms_reg.reg_plain (id bigint, value string) using parquet + location '${root}/elsewhere/plain'""") + runSpark("insert into delta_hms_reg.reg_plain values (1, 'parquet')") + runSpark("create view delta_hms_reg.reg_view as select id from delta_hms_reg.reg_external") + runSpark("""create table delta_hms_reg_plain.plain_only (id bigint) using parquet + location '${root}/elsewhere/plain_only'""") + + // Changed by Spark below, after both catalogs have cached them. + runSpark("""create table delta_hms_reg.reg_rename_external (id bigint, value string) using delta + location '${root}/elsewhere/rename_external' tblproperties ('delta.enableDeletionVectors'='true')""") + runSpark("""create table delta_hms_reg.reg_rename_managed (id bigint, value string) using delta + tblproperties ('delta.enableDeletionVectors'='true')""") + for (String table : ["reg_rename_external", "reg_rename_managed"]) { + runSpark("insert into delta_hms_reg.${table} values (1, 'before'), (2, 'before')") + } + runSpark("""create table delta_hms_reg.reg_drop_external (id bigint) using delta + location '${root}/elsewhere/drop_external'""") + runSpark("insert into delta_hms_reg.reg_drop_external values (1)") + for (String table : ["reg_drop_managed", "reg_recreate", "reg_replace"]) { + runSpark("create table delta_hms_reg.${table} (id bigint) using delta") + runSpark("insert into delta_hms_reg.${table} values (1)") + } + runSpark("create table delta_hms_reg_dropped.reg_in_dropped_db (id bigint) using delta") + runSpark("insert into delta_hms_reg_dropped.reg_in_dropped_db values (1)") + + for (String label : catalogTypes.keySet()) { + String typeProperties = catalogTypes[label] + String catalog = "test_deltalake_hms_reg_${label}" + createCatalog(catalog, typeProperties, "") + + // Discovery. The gateway also lists non-Delta objects, the dedicated catalog must not. + "order_qt_${label}_databases"("show databases from ${catalog} like 'delta_hms_reg%'") + "order_qt_${label}_tables"("show tables from ${catalog}.delta_hms_reg") + "order_qt_${label}_plain_db_tables"("show tables from ${catalog}.delta_hms_reg_plain") + sql "switch ${catalog}" + "order_qt_${label}_info_tables"("""select TABLE_NAME from information_schema.tables + where TABLE_SCHEMA = 'delta_hms_reg' and TABLE_NAME like 'reg_m%'""") + "order_qt_${label}_info_columns"("""select COLUMN_NAME, DATA_TYPE from information_schema.columns + where TABLE_SCHEMA = 'delta_hms_reg' and TABLE_NAME = 'reg_managed'""") + sql "switch internal" + + // Registration forms. + for (String table : ["reg_managed", "reg_ctas", "reg_external", "reg_no_columns", "reg_space_path", + "reg_trailing_slash", "reg_mixed_case"]) { + "qt_${label}_${table}_schema"("desc ${catalog}.delta_hms_reg.${table}") + "order_qt_${label}_${table}"("select * from ${catalog}.delta_hms_reg.${table}") + "qt_${label}_${table}_count"("select count(*) from ${catalog}.delta_hms_reg.${table}") + } + "qt_${label}_external_ddl"("show create table ${catalog}.delta_hms_reg.reg_external") + "order_qt_${label}_managed_partition"( + "select * from ${catalog}.delta_hms_reg.reg_managed where part = 'a'") + "order_qt_${label}_managed_null_partition"( + "select * from ${catalog}.delta_hms_reg.reg_managed where part is null") + "order_qt_${label}_mixed_case_column"( + "select ID, mixedvalue from ${catalog}.delta_hms_reg.reg_mixed_case where MIXEDVALUE = 'mixed'") + + // A non-Delta table is invisible to the dedicated catalog and stays plain Hive through the gateway. + if (label == "dedicated") { + test { + sql "select * from ${catalog}.delta_hms_reg.reg_plain" + exception "Table [reg_plain] does not exist" + } + } else { + "order_qt_${label}_plain"("select * from ${catalog}.delta_hms_reg.reg_plain") + } + + // The gateway must reject writes before Hive can change the Delta table's data or HMS registration. + test { + sql "insert into ${catalog}.delta_hms_reg.reg_external values (3, 'rejected')" + exception "Connector does not provide a write plan for target reg_external" + } + test { + sql "truncate table ${catalog}.delta_hms_reg.reg_external" + exception "TRUNCATE TABLE not supported" + } + test { + sql "alter table ${catalog}.delta_hms_reg.reg_external rename reg_external_rejected" + exception "RENAME TABLE not supported" + } + test { + sql "drop table ${catalog}.delta_hms_reg.reg_external" + exception "DROP TABLE not supported" + } + "order_qt_${label}_external_after_rejected_writes"("select * from ${catalog}.delta_hms_reg.reg_external") + + // Cache every table that Spark changes below. + for (String table : ["reg_rename_external", "reg_rename_managed", "reg_drop_external", "reg_drop_managed", + "reg_recreate", "reg_replace"]) { + "order_qt_${label}_${table}_initial"("select * from ${catalog}.delta_hms_reg.${table}") + } + "order_qt_${label}_in_dropped_db_initial"( + "select * from ${catalog}.delta_hms_reg_dropped.reg_in_dropped_db") + } + + // Table names are case-insensitive only when the catalog says so. Database names never are in these modes. + for (String label : catalogTypes.keySet()) { + String typeProperties = catalogTypes[label] + for (String mode : ["0", "1", "2"]) { + String catalog = "test_deltalake_hms_reg_${label}_case_${mode}" + createCatalog(catalog, typeProperties, "'only_test_lower_case_table_names'='${mode}',") + "order_qt_${label}_case_${mode}_lower"("select * from ${catalog}.delta_hms_reg.reg_mixed_case") + "order_qt_${label}_case_${mode}_tables"( + "show tables from ${catalog}.delta_hms_reg like '%MIXED_CASE%'") + if (mode == "0") { + test { + sql "select * from ${catalog}.delta_hms_reg.REG_MIXED_CASE" + exception "Table [REG_MIXED_CASE] does not exist in database [delta_hms_reg]" + } + } else { + "order_qt_${label}_case_${mode}_upper"("select * from ${catalog}.delta_hms_reg.REG_MIXED_CASE") + } + test { + sql "select * from ${catalog}.DELTA_HMS_REG.reg_mixed_case" + exception "Database [DELTA_HMS_REG] does not exist" + } + } + } + + // Renames. The external path stays put, so the name stops matching the directory. HMS moves the managed + // directory and Spark rewrites the SerDe path. New commits prove that the new name reaches the live log. + for (String table : ["reg_rename_external", "reg_rename_managed"]) { + runSpark("alter table delta_hms_reg.${table} rename to delta_hms_reg.${table}_new") + runSpark("delete from delta_hms_reg.${table}_new where id = 1") + runSpark("insert into delta_hms_reg.${table}_new values (3, 'after')") + } + // Dropping external tables keeps the files, so the same path can be registered again under another name. + runSpark("drop table delta_hms_reg.reg_drop_external") + runSpark("""create table delta_hms_reg.reg_reattached using delta + location '${root}/elsewhere/drop_external'""") + runSpark("drop table delta_hms_reg.reg_drop_managed") + // Same name, new table. DROP removes the managed files, while REPLACE commits a new schema on the same path. + runSpark("drop table delta_hms_reg.reg_recreate") + runSpark("create table delta_hms_reg.reg_recreate (recreated string) using delta") + runSpark("insert into delta_hms_reg.reg_recreate values ('recreated')") + runSpark("create or replace table delta_hms_reg.reg_replace (replaced string) using delta") + runSpark("insert into delta_hms_reg.reg_replace values ('replaced')") + // One commit through one name must reach the other name of the same path. + runSpark("insert into delta_hms_reg.reg_no_columns values (3, 'external-three')") + runSpark("create table delta_hms_reg.reg_added_later (id bigint) using delta") + runSpark("insert into delta_hms_reg.reg_added_later values (1)") + runSpark("drop database delta_hms_reg_dropped cascade") + + for (String label : catalogTypes.keySet()) { + String catalog = "test_deltalake_hms_reg_${label}" + sql "refresh database ${catalog}.delta_hms_reg" + "order_qt_${label}_tables_after_changes"("show tables from ${catalog}.delta_hms_reg") + + for (String table : ["reg_rename_external", "reg_rename_managed"]) { + test { + sql "select * from ${catalog}.delta_hms_reg.${table}" + exception "Table [${table}] does not exist" + } + "qt_${label}_${table}_new_schema"("desc ${catalog}.delta_hms_reg.${table}_new") + "order_qt_${label}_${table}_new"("select * from ${catalog}.delta_hms_reg.${table}_new") + "qt_${label}_${table}_new_count"("select count(*) from ${catalog}.delta_hms_reg.${table}_new") + } + for (String table : ["reg_drop_external", "reg_drop_managed"]) { + test { + sql "select * from ${catalog}.delta_hms_reg.${table}" + exception "Table [${table}] does not exist" + } + } + "qt_${label}_reattached_schema"("desc ${catalog}.delta_hms_reg.reg_reattached") + "order_qt_${label}_reattached"("select * from ${catalog}.delta_hms_reg.reg_reattached") + for (String table : ["reg_recreate", "reg_replace"]) { + "qt_${label}_${table}_schema_after_change"("desc ${catalog}.delta_hms_reg.${table}") + "order_qt_${label}_${table}_after_change"("select * from ${catalog}.delta_hms_reg.${table}") + } + for (String table : ["reg_external", "reg_no_columns"]) { + "order_qt_${label}_${table}_shared_path"("select * from ${catalog}.delta_hms_reg.${table}") + } + "order_qt_${label}_added_later"("select * from ${catalog}.delta_hms_reg.reg_added_later") + + sql "refresh catalog ${catalog}" + "order_qt_${label}_databases_after_drop"("show databases from ${catalog} like 'delta_hms_reg%'") + test { + sql "select * from ${catalog}.delta_hms_reg_dropped.reg_in_dropped_db" + exception "Database [delta_hms_reg_dropped] does not exist" + } + } + } finally { + spark.close() + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_metadata.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_metadata.groovy new file mode 100644 index 00000000000000..2c1eb955cf5714 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_metadata.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.JdbcUtils + +import java.sql.DriverManager + +suite("test_deltalake_metadata", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String sparkPort = context.config.otherConfigs.get("deltalakeSparkPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + Class.forName("org.apache.hive.jdbc.HiveDriver") + def spark = DriverManager.getConnection("jdbc:hive2://${host}:${sparkPort}/default", "hadoop", "hadoop") + try { + sql "set num_files_in_batch_mode=1" + for (String source : ["filesystem", "hms"]) { + String catalog = "test_deltalake_metadata_${source}" + String metastore = source == "filesystem" ? "'warehouse'='${warehouse}'" + : "'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + for (String suffix : ["", "_uncached"]) { + sql "drop catalog if exists ${catalog}${suffix}" + sql """create catalog ${catalog}${suffix} properties ( + 'type'='deltalake', 'deltalake.catalog.type'='${source}', ${metastore}, + 'meta.cache.deltalake.latest.ttl-second'='${suffix.isEmpty() ? -1 : 0}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + } + for (boolean streaming : [false, true]) { + sql "set enable_external_table_batch_mode=${streaming}" + String label = "${source}_${streaming}" + for (String table : ["multi_part_checkpoint", "only_checkpoint_files", "v2_checkpoint_json", + "v2_checkpoint_parquet", "basic_with_vacuum_protocol_check_feature", + "dv_partitioned_with_checkpoint", "dv_with_columnmapping"]) { + "order_qt_${label}_${table}"("select * from ${catalog}.delta_golden.${table}") + "qt_${label}_${table}_count"("select count(*) from ${catalog}.delta_golden.${table}") + } + "order_qt_${label}_compaction"("select * from ${catalog}.delta_m1.log_compaction") + "order_qt_${label}_pruning"("select * from ${catalog}.delta_m1.skipping where id<5") + "qt_${label}_count"("select count(*) from ${catalog}.delta_m1.skipping") + sql "set file_split_size=128" + "order_qt_${label}_dv_splits"("select * from ${catalog}.delta_m1.dv") + "order_qt_${label}_mapping_splits"("select * from ${catalog}.delta_m1.mapping_id") + sql "set file_split_size=0" + } + // Fixed, unlimited TTL makes refresh checks independent of machine speed. + // The fake-clock unit test separately proves that ordinary cache hits never extend a finite TTL. + JdbcUtils.executeToList(spark, """create or replace table delta_m1.refresh_${source} (id bigint) + using delta tblproperties ('delta.enableDeletionVectors'='false')""") + JdbcUtils.executeToList(spark, "insert into delta_m1.refresh_${source} values (1)") + "order_qt_${source}_initial"("select * from ${catalog}.delta_m1.refresh_${source}") + JdbcUtils.executeToList(spark, "insert into delta_m1.refresh_${source} values (2)") + "order_qt_${source}_cached"("select * from ${catalog}.delta_m1.refresh_${source}") + "order_qt_${source}_uncached"("select * from ${catalog}_uncached.delta_m1.refresh_${source}") + sql "refresh table ${catalog}.delta_m1.refresh_${source}" + "order_qt_${source}_table_refresh"("select * from ${catalog}.delta_m1.refresh_${source}") + JdbcUtils.executeToList(spark, "alter table delta_m1.refresh_${source} add columns (value string)") + JdbcUtils.executeToList(spark, "insert into delta_m1.refresh_${source} values (3, 'new-schema')") + "qt_${source}_cached_schema"("desc ${catalog}.delta_m1.refresh_${source}") + sql "refresh table ${catalog}.delta_m1.refresh_${source}" + "qt_${source}_refreshed_schema"("desc ${catalog}.delta_m1.refresh_${source}") + "order_qt_${source}_schema_rows"("select * from ${catalog}.delta_m1.refresh_${source}") + JdbcUtils.executeToList(spark, "insert into delta_m1.refresh_${source} values (4, 'database-refresh')") + sql "refresh database ${catalog}.delta_m1" + "order_qt_${source}_database_refresh"("select * from ${catalog}.delta_m1.refresh_${source}") + JdbcUtils.executeToList(spark, "insert into delta_m1.refresh_${source} values (5, 'catalog-refresh')") + sql "refresh catalog ${catalog}" + "order_qt_${source}_catalog_refresh"("select * from ${catalog}.delta_m1.refresh_${source}") + } + } finally { + spark.close() + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_metrics.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_metrics.groovy new file mode 100644 index 00000000000000..a12c53cd4b795c --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_metrics.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("test_deltalake_metrics", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "drop catalog if exists test_deltalake_metrics" + sql """create catalog test_deltalake_metrics properties ( + 'type'='deltalake', 'warehouse'='${warehouse}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "switch test_deltalake_metrics" + sql "use delta_m1" + sql "set enable_profile=true" + sql "set enable_sql_cache=false" + sql "set enable_count_push_down_for_external_table=true" + sql "set num_files_in_batch_mode=1" + def profiles = new ProfileAction(context) + [false, true].each { boolean streaming -> + sql "set enable_external_table_batch_mode=${streaming}" + String token = UUID.randomUUID().toString() + "order_qt_filter_${streaming}"("select /* ${token} */ id, s from skipping where id < 5 order by id") + String profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + // Two files survive Kernel, including the all-null file with absent min/max statistics. + ["TotalFiles: 3", "SelectedFiles: 2", "SelectedPartitions: 0", "GeneratedSplits: 2"].each { + assertTrue(profile.contains(it), "Missing ${it}: ${profile}") + } + explain { + sql "select id, s from skipping where id < 5" + contains "Delta snapshot:" + contains "Delta files:" + if (!streaming) { + contains "Delta files: 2/3, selected partitions: 0, splits: 2" + } + } + token = UUID.randomUUID().toString() + "qt_count_${streaming}"("select /* ${token} */ count(*) from skipping") + profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + ["TotalFiles: 3", "SelectedFiles: 3", "GeneratedSplits: 1"].each { + assertTrue(profile.contains(it), "Missing ${it}: ${profile}") + } + token = UUID.randomUUID().toString() + "order_qt_partition_${streaming}"("""select /* ${token} */ renamed + from mapping_name where partition_renamed='a/b' order by renamed""") + profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + ["TotalFiles: 2", "SelectedFiles: 1", "SelectedPartitions: 1", "GeneratedSplits: 1"].each { + assertTrue(profile.contains(it), "Missing ${it}: ${profile}") + } + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet.groovy new file mode 100644 index 00000000000000..61765224b7ecab --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("test_deltalake_parquet", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + def profileAction = new ProfileAction(context) + def counterValues = { String profile, String counter -> + def values = [] + def matcher = profile =~ ("(?m)^\\s*(?:-\\s*)?" + + java.util.regex.Pattern.quote(counter) + ":\\s+([^\\n]+)") + while (matcher.find()) { + String text = matcher.group(1) + def exact = text =~ /\(([0-9,]+)\)/ + def number = text =~ /([0-9,]+)/ + values.add(Long.parseLong((exact.find() ? exact.group(1) : number.find() ? number.group(1) : "0") + .replace(",", ""))) + } + return values + } + sql "set enable_profile=true" + sql "set enable_sql_cache=false" + ["filesystem", "hms"].each { String metastore -> + String catalog = "test_deltalake_parquet_${metastore}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + 'type'='deltalake', 'deltalake.catalog.type'='${metastore}', 'warehouse'='${warehouse}', + 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + ["none", "name", "id"].each { String mapping -> + String column = mapping == "none" ? "id" : "renamed" + String table = "${catalog}.delta_m1.parquet_pruning_${mapping}" + String predicate = "${column} >= 100 and ${column} < 120" + // The one file overlaps this predicate, so Kernel cannot prune the file itself. + explain { + sql "select ${column}, length(payload) from ${table} where ${predicate}" + contains "inputSplitNum=1" + } + String token = UUID.randomUUID().toString() + "order_qt_${metastore}_${mapping}"(""" + select /* ${token} */ ${column}, length(payload) + from ${table} where ${predicate} order by ${column} + """) + String profile = profileAction.getProfileBySql(token, + ["RowGroupsFilteredByMinMax", "FilteredRowsByPage", "PageIndexReadCalls"]) + ["RowGroupsFilteredByMinMax", "FilteredRowsByPage", "PageIndexReadCalls"].each { String counter -> + assertTrue(counterValues(profile, counter).sum(0L) > 0L, + "${metastore}/${mapping}: ${counter} did not show Parquet pruning; ${profile}") + } + } + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.groovy new file mode 100644 index 00000000000000..e5cb4cb1dd00fa --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_parquet_deletion_vector.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +// A deletion vector addresses rows by their position in the whole file. Row group and page pruning skip rows before +// the delete predicate runs, so the surviving rows must keep those file positions: if id 101 is deleted and the first +// page was skipped, counting from the first row read would delete id 201 instead. The table has one file whose ids +// equal their positions, which makes such a shift visible in the results. +// +// The profile counters prove that pruning and the deletion vector were both active in the same scan. The abs() +// queries return the same ranges through a predicate that Parquet statistics cannot evaluate, so a pruning-only +// difference shows up between neighbouring result blocks. +suite("test_deltalake_parquet_deletion_vector", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + def profileAction = new ProfileAction(context) + def counterValues = { String profile, String counter -> + def values = [] + def matcher = profile =~ ("(?m)^\\s*(?:-\\s*)?" + + java.util.regex.Pattern.quote(counter) + ":\\s+([^\\n]+)") + while (matcher.find()) { + String text = matcher.group(1) + def exact = text =~ /\(([0-9,]+)\)/ + def number = text =~ /([0-9,]+)/ + values.add(Long.parseLong((exact.find() ? exact.group(1) : number.find() ? number.group(1) : "0") + .replace(",", ""))) + } + return values + } + List pageCounters = ["FilteredRowsByPage", "PageIndexReadCalls", "NumDeleteRows"] + List rangeCounters = pageCounters + ["RowGroupsFilteredByMinMax"] + sql "set enable_profile=true" + sql "set enable_sql_cache=false" + ["filesystem", "hms"].each { String metastore -> + String catalog = "test_deltalake_parquet_dv_${metastore}" + String table = "${catalog}.delta_m1.parquet_pruning_dv" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + 'type'='deltalake', 'deltalake.catalog.type'='${metastore}', 'warehouse'='${warehouse}', + 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + + def checkPrunedRows = { String name, String predicate, List counters -> + String token = UUID.randomUUID().toString() + "order_qt_${metastore}_${name}"(""" + select /* ${token} */ id, payload from ${table} where ${predicate} order by id + """) + String profile = profileAction.getProfileBySql(token, + counters + ["Delta Scan Metrics", "EnumerationComplete: true"]) + assertEquals([1L], counterValues(profile, "SelectedFiles"), profile) + counters.each { String counter -> + assertTrue(counterValues(profile, counter).any { it > 0 }, + "${metastore}/${name}: expected positive ${counter}; ${profile}") + } + return profile + } + + // First row group, around the start of the second page of the id column, near the page edge at about id 200, + // and a row group far from the start of the file. + Map> ranges = ["early": [100, 120], "page_edge": [195, 205], "late": [9000, 9020]] + ranges.each { String range, List bounds -> + int lower = bounds[0] + int upper = bounds[1] + String profile = checkPrunedRows(range, "id >= ${lower} and id < ${upper}", rangeCounters) + assertEquals([1L], counterValues(profile, "GeneratedSplits").unique(), profile) + "order_qt_${metastore}_${range}_no_pruning"(""" + select id, payload + from ${table} where abs(id) >= ${lower} and abs(id) < ${upper} order by id + """) + "qt_${metastore}_${range}_count"( + "select count(*) from ${table} where id >= ${lower} and id < ${upper}") + } + + // On the first selected page only id 5 matches, and it is deleted. Continue past the gap for 201 and 204. + String disjoint = checkPrunedRows("disjoint_pages", "id in (5, 200, 201, 204)", + rangeCounters + ["SelectedRowRanges", "RowGroupsReadNum"]) + assertEquals(1L, counterValues(disjoint, "RowGroupsReadNum").max(), disjoint) + assertTrue(counterValues(disjoint, "SelectedRowRanges").max() >= 2, disjoint) + + String deleted = "id >= 199 and id < 201" + checkPrunedRows("all_deleted", deleted, rangeCounters) + "qt_${metastore}_all_deleted_aggregates"( + "select count(*), min(id), max(id) from ${table} where ${deleted}") + + def previousSplitSize = sql("select @@file_split_size")[0][0] + try { + sql "set file_split_size=32768" + String predicate = "id in (101, 102, 9005, 9006)" + // Deleted positions 101 and 9005 still refer to the whole file when its later split starts. + String splitProfile = checkPrunedRows("cross_split", predicate, pageCounters + ["RowGroupsReadNum"]) + assertTrue(counterValues(splitProfile, "GeneratedSplits").min() > 1, splitProfile) + assertTrue(counterValues(splitProfile, "RowGroupsReadNum").max() >= 2, splitProfile) + } finally { + sql "set file_split_size=${previousSplitSize}" + } + + // Footer statistics still count deleted rows. Ids 0 and 9999 are deleted, so MIN and MAX must not use them. + "qt_${metastore}_count"("select count(*) from ${table}") + "qt_${metastore}_min_max"("select min(id), max(id) from ${table}") + "qt_${metastore}_misaligned_payloads"(""" + select count(*) from ${table} where not (payload <=> rpad(concat('row-', cast(id as string)), 64, 'x')) + """) + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_scan_reuse.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_scan_reuse.groovy new file mode 100644 index 00000000000000..443459c66ac485 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_scan_reuse.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("test_deltalake_scan_reuse", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "set enable_sql_cache=false" + sql "set enable_profile=true" + sql "set num_files_in_batch_mode=1" + def profiles = new ProfileAction(context) + + for (String source : ["filesystem", "hms", "gateway"]) { + String catalog = "test_deltalake_scan_reuse_${source}" + String type = source == "gateway" ? "hms" : "deltalake" + String metastore = source == "filesystem" ? "filesystem" : "hms" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + 'type'='${type}', 'deltalake.catalog.type'='${metastore}', 'warehouse'='${warehouse}', + 'hive.metastore.uris'='thrift://${host}:${hmsPort}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + String table = "${catalog}.delta_m1.parquet_pruning_dv" + for (String mode : ["off", "on", "stream"]) { + sql "set enable_external_scan_task_reuse=${mode != 'off'}" + sql "set enable_external_table_batch_mode=${mode == 'stream'}" + // The first two scans share their typed predicate. The third must keep its own ranges. + // Payload embeds the id, so this also detects DV/range descriptor corruption on reuse. + String token = UUID.randomUUID().toString() + "order_qt_${source}_${mode}_ranges"(""" + select /* ${token} */ 'first' as branch, id, payload from ${table} where id >= 100 and id < 107 + union all + select 'second', id, payload from ${table} where id >= 100 and id < 107 + union all + select 'different', id, payload from ${table} where id >= 195 and id < 201 + """) + String profile = profiles.getProfileBySql(token, ["Delta Scan Metrics", "EnumerationComplete: true"]) + if (mode == "on") { + assertTrue(profile.contains("PlanningReuseHit: true"), profile) + } else { + assertFalse(profile.contains("PlanningReuseHit: true"), profile) + } + "order_qt_${source}_${mode}_count"(""" + select 'first' as branch, count(*) from ${table} + union all select 'second', count(*) from ${table} + union all select 'filtered', count(*) from ${table} where id >= 100 and id < 107 + """) + "order_qt_${source}_${mode}_mapping"(""" + select 'first' as branch, * from ${catalog}.delta_m1.mapping_id + union all select 'second', * from ${catalog}.delta_m1.mapping_id + """) + } + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_skipping.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_skipping.groovy new file mode 100644 index 00000000000000..caa957bbfd4d46 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_skipping.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_deltalake_skipping", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "drop catalog if exists test_deltalake_skipping" + sql """create catalog test_deltalake_skipping properties ( + 'type'='deltalake', 'warehouse'='${warehouse}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "switch test_deltalake_skipping" + sql "use delta_m1" + sql "set enable_count_push_down_for_external_table=true" + order_qt_all "select * from skipping order by id" + order_qt_range "select * from skipping where id < 5 order by id" + order_qt_reversed "select * from skipping where 5 > id order by id" + order_qt_between "select * from skipping where id >= 5 and id <= 7 order by id" + order_qt_or "select * from skipping where id < 2 or id > 107 order by id" + order_qt_null "select * from skipping where id is null order by id" + order_qt_not_null "select * from skipping where id is not null order by id" + order_qt_not_null_safe "select * from skipping where not (id <=> 1) order by id" + order_qt_in "select * from skipping where id in (1, 101, null) order by id" + order_qt_not_in "select * from skipping where id not in (1, 101, null) order by id" + order_qt_partial_and "select * from skipping where id < 5 and substring(s, 1, 3)='low' order by id" + order_qt_partial_or "select * from skipping where id > 107 or substring(s, 1, 3)='low' order by id" + order_qt_partial_not "select * from skipping where not (id > 107 and substring(s, 1, 4)='high') order by id" + order_qt_decimal "select * from skipping where dec >= 100.0000 order by id" + order_qt_float "select * from skipping where f > 107.5 order by id" + order_qt_double "select * from skipping where d > 107.5 order by id" + qt_count "select count(*) from skipping" + qt_count_column "select count(id) from skipping" + qt_filtered_count "select count(*) from skipping where id < 5" + qt_function_count "select count(*) from skipping where substring(s, 1, 3)='low'" + explain { + sql "select * from skipping" + contains "inputSplitNum=3" + contains "Delta snapshot:" + } + explain { + sql "select * from skipping where id < 5" + // Kernel conservatively retains the all-null file, whose min/max are absent. + contains "inputSplitNum=2" + } + explain { + sql "select * from skipping where id is null" + contains "inputSplitNum=1" + } + explain { + sql "select * from skipping where id is not null" + contains "inputSplitNum=2" + } + explain { + sql "select count(*) from skipping" + contains "inputSplitNum=1" + contains "pushdown agg=COUNT (21)" + } + sql "set enable_count_push_down_for_external_table=false" + qt_count_without_pushdown "select count(*) from skipping" +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_statistics.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_statistics.groovy new file mode 100644 index 00000000000000..da24a39a6f6e6a --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_statistics.groovy @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_deltalake_statistics", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + for (String source : ["filesystem", "hms", "hive"]) { + String catalog = "test_deltalake_statistics_${source}" + sql "drop catalog if exists ${catalog}" + String type = source == "hive" ? "hms" : "deltalake" + String metastore = source == "filesystem" ? "'warehouse'='${warehouse}'" + : "'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + String mode = source == "hive" ? "" : "'deltalake.catalog.type'='${source}'," + sql """create catalog ${catalog} properties ( + 'type'='${type}', ${mode} ${metastore}, + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + for (String table : ["delta_m1.basic", "delta_golden.dv_partitioned_with_checkpoint", + "delta_golden.v2_checkpoint_json", "delta_golden.v2_checkpoint_parquet"]) { + String label = "${source}_${table.replace('.', '_')}" + "qt_${label}_count"("select count(*) from ${catalog}.${table}") + } + sql "refresh table ${catalog}.delta_m1.basic" + "qt_${source}_refreshed_count"("select count(*) from ${catalog}.delta_m1.basic") + "qt_${source}_empty_count"("select count(*) from ${catalog}.delta_m1.empty") + sql "alter catalog ${catalog} set properties ('meta.cache.deltalake.statistics.enable'='false')" + // With collection disabled, the three small Parquet files yield a CRC size estimate above three rows. + explain { + sql("select * from ${catalog}.delta_m1.basic") + check { plan -> + def rows = plan =~ /\bcardinality=(\d+)\b/ + rows.find() && rows.group(1).toLong() > 3 + } + } + // The golden table has no version checksum. CBO uses its unknown-row fallback and COUNT is exact. + explain { + sql("select * from ${catalog}.delta_golden.dv_partitioned_with_checkpoint") + check { plan -> (plan =~ /\bcardinality=1\b/).find() } + } + "qt_${source}_disabled_count"("select count(*) from ${catalog}.delta_golden.dv_partitioned_with_checkpoint") + } +} diff --git a/regression-test/suites/external_table_p0/deltalake/test_deltalake_version_as_of.groovy b/regression-test/suites/external_table_p0/deltalake/test_deltalake_version_as_of.groovy new file mode 100644 index 00000000000000..4c3e7a6070b026 --- /dev/null +++ b/regression-test/suites/external_table_p0/deltalake/test_deltalake_version_as_of.groovy @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.JdbcUtils + +import java.sql.DriverManager +import java.sql.Types + +suite("test_deltalake_version_as_of", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String sparkPort = context.config.otherConfigs.get("deltalakeSparkPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + String table = "delta_version_as_of.`history`" + Map catalogs = [ + filesystem: "'type'='deltalake', 'deltalake.catalog.type'='filesystem', 'warehouse'='${warehouse}'", + hms: "'type'='deltalake', 'deltalake.catalog.type'='hms', 'hive.metastore.uris'='thrift://${host}:${hmsPort}'", + gateway: "'type'='hms', 'hive.metastore.uris'='thrift://${host}:${hmsPort}'", + ] + Class.forName("org.apache.hive.jdbc.HiveDriver") + def spark = DriverManager.getConnection("jdbc:hive2://${host}:${sparkPort}/default", "hadoop", "hadoop") + try { + def sparkSql = { String statement -> JdbcUtils.executeToList(spark, statement) } + def version = { String name = table -> sparkSql("describe history ${name} limit 1")[0][0][0].toLong() } + + // Spark enforces retention on time travel. Create the commits for this run and keep them for inspection. + sparkSql("create database if not exists delta_version_as_of location '${warehouse}/delta_version_as_of'") + sparkSql("drop table if exists ${table}") + sparkSql("""create table ${table} (id bigint, payload string, part string) + using delta partitioned by (part) + tblproperties ('delta.columnMapping.mode'='name', 'delta.enableDeletionVectors'='true')""") + sparkSql("""insert into ${table} select /*+ COALESCE(1) */ id, concat('row-', id), + case when id=0 then 'gone' when id=7 then null else 'a/b' end from range(20)""") + long inserted = version() + + // Reusing a logical name assigns a new column ID. Keep the original Parquet file across the change. + String idMapped = "delta_version_as_of.readded_id" + sparkSql("drop table if exists ${idMapped}") + sparkSql("""create table ${idMapped} (id bigint, payload string) using delta + tblproperties ('delta.columnMapping.mode'='id', 'delta.enableDeletionVectors'='false')""") + sparkSql("insert into ${idMapped} values (1, 'old-value')") + long originalColumn = version(idMapped) + sparkSql("alter table ${idMapped} drop column payload") + long droppedColumn = version(idMapped) + sparkSql("alter table ${idMapped} add columns (payload bigint)") + sparkSql("insert into ${idMapped} values (2, 10000000000)") + long newColumn = version(idMapped) + + sql "set enable_sql_cache=false" + catalogs.each { source, properties -> + String catalog = "test_deltalake_version_as_of_${source}" + sql "drop catalog if exists ${catalog}" + sql """create catalog ${catalog} properties ( + ${properties}, 'meta.cache.deltalake.latest.ttl-second'='-1', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + "qt_${source}_warm_latest"("select count(*) from ${catalog}.${table}") + } + + sparkSql("delete from ${table} where id in (2,3)") + long deleted = version() + // Both deleted rows share a partially retained data file. Check that Spark actually wrote a DV. + String commit = String.format("%020d.json", deleted) + def dvFiles = sparkSql("""select count(*) from json.`${warehouse}/delta_version_as_of/history/_delta_log/${commit}` + where add.deletionVector.cardinality > 0""")[0] + assertTrue(dvFiles[0][0].toLong() > 0, "Spark did not write a deletion vector at version ${deleted}") + sparkSql("delete from ${table} where part='gone'") + sparkSql("alter table ${table} rename column payload to label") + sparkSql("alter table ${table} rename column part to segment") + sparkSql("insert into ${table} values (20, 'new-20', 'new'), (21, 'new-21', 'new')") + long evolved = version() + + // Both drivers expose BIGINT directly. Hive STRING and Doris STRING can use different JDBC character types. + def schema = { meta -> + (1..meta.columnCount).collect { i -> + int type = meta.getColumnType(i) + [meta.getColumnLabel(i).tokenize('.').last(), + type in [Types.CHAR, Types.VARCHAR, Types.LONGVARCHAR] ? Types.VARCHAR : type] + } + } + def compareWithSpark = { String goldenTag = "", String comparedTable = table, Closure query -> + def (expected, sparkMeta) = sparkSql(query(comparedTable, "version as of")) + catalogs.each { source, ignored -> + String name = "test_deltalake_version_as_of_${source}.${comparedTable}" + String statement = query(name, "for version as of") + def (actual, dorisMeta) = JdbcUtils.executeToList(context.getConnection(), statement) + assertEquals(schema(sparkMeta), schema(dorisMeta), statement) + assertSparkDorisResultEquals(expected, actual) + if (goldenTag) { + quickTest("${source}_${goldenTag}", statement) + } + } + } + + [original: originalColumn, dropped: droppedColumn, readded: newColumn].each { stage, snapshot -> + compareWithSpark("id_${stage}", idMapped) { name, at -> + "select * from ${name} ${at} ${snapshot} order by id" + } + } + for (long snapshot : [0L, inserted, deleted, evolved]) { + compareWithSpark { name, at -> "select * from ${name} ${at} ${snapshot} order by id" } + } + compareWithSpark("dropped_partition") { name, at -> + "select id, payload from ${name} ${at} ${inserted} where part='gone' order by id" + } + compareWithSpark { name, at -> + "select id, label from ${name} ${at} ${evolved} where segment is null or segment='new' order by id" + } + compareWithSpark { name, at -> """select o.id, o.payload, n.label + from ${name} ${at} ${inserted} o left join ${name} ${at} ${evolved} n on o.id=n.id + where o.id <= 4 order by o.id""" } + // COUNT with and without a filter exercises metadata counting and decoding rows masked by the DV. + for (boolean streaming : [false, true]) { + sql "set enable_external_table_batch_mode=${streaming}" + compareWithSpark(streaming ? "" : "dv_counts") { name, at -> + """select 'before' as phase, count(*) as n from ${name} ${at} ${inserted} + union all select 'after', count(*) from ${name} ${at} ${deleted} + union all select 'masked', count(*) from ${name} ${at} ${deleted} where id in (2,3) + order by phase""" } + } + + catalogs.each { source, ignored -> + String name = "test_deltalake_version_as_of_${source}.${table}" + // Explicit lookups above succeeded for versions newer than the warmed, non-expiring latest pin. + "qt_${source}_cached_latest"("select count(*) from ${name}") + "qt_${source}_mixed_latest"("""select a.payload, b.label from ${name} a + left join ${name} for version as of '${evolved}' b on a.id=b.id where a.id=2""") + test { + sql "select label from ${name} for version as of ${inserted}" + exception "Unknown column 'label'" + } + test { + sql "select payload from ${name} for version as of ${evolved}" + exception "Unknown column 'payload'" + } + long future = evolved + 1 + test { + sql "select * from ${name} for version as of ${future}" + exception "Cannot load table version ${future} as it does not exist" + } + for (String badVersion : ["not_a_version", "-1", "9223372036854775808"]) { + test { + sql "select * from ${name} for version as of '${badVersion}'" + exception "Delta Lake FOR VERSION AS OF requires a non-negative 64-bit integer" + } + } + sql "refresh table ${name}" + "qt_${source}_refreshed_latest"("select count(*) from ${name}") + } + } finally { + spark.close() + } +} diff --git a/regression-test/suites/external_table_p2/deltalake/README.md b/regression-test/suites/external_table_p2/deltalake/README.md new file mode 100644 index 00000000000000..eb0076b454337c --- /dev/null +++ b/regression-test/suites/external_table_p2/deltalake/README.md @@ -0,0 +1,262 @@ +# Delta Lake p2 acceptance + +## Storage backends + +`test_deltalake_storage` runs the same expected results against one storage +backend at a time. Prepare the pinned golden tables with +`docker/thirdparties/docker-compose/iceberg/scripts/deltalake/prepare.py`, then +copy its `cache/deltalake/golden/` contents into +`/delta_golden/` on the target storage. Preserve every table's +relative paths, including `_delta_log`, checkpoint sidecars and DV files. + +For example, with an existing HDFS cluster and its Hadoop client configuration: + +```bash +hdfs dfs -mkdir -p /warehouse/delta_golden +hdfs dfs -put -f docker/thirdparties/docker-compose/iceberg/cache/deltalake/golden/* \ + /warehouse/delta_golden/ +``` + +Create a local JSON file containing the catalog's warehouse and normal +`fe-filesystem` storage properties, for example: + +```json +{ + "warehouse": "hdfs://namenode:8020/warehouse", + "fs.defaultFS": "hdfs://namenode:8020", + "hadoop.username": "hadoop" +} +``` + +Keep credentials and environment-specific files outside version control. Set +these values in the local regression config: + +```groovy +enableDeltaLakeTest=true +enableDeltaLakeStorageTest=true +deltalakeStoragePropertiesFile="/absolute/path/to/storage-properties.json" +``` + +Then run: + +```bash +./run-regression-test.sh --run -d external_table_p2/deltalake -s test_deltalake_storage +``` + +The suite covers column mapping, timestamps, nested type widening, disk DVs, +partitioned checkpoints and both V2 checkpoint formats. Synchronous and streaming +planning read 128-byte splits, including splits sharing a file-global DV. It +uses the same `.out` for each backend; an enabled but unavailable or misconfigured +backend fails the suite rather than being skipped. + +Local acceptance on 2026-10-30 passed against both MinIO and ordinary HDFS 3.2.1. +OSS, COS, OBS, Azure, GCS and Kerberos HDFS remain unverified pending test +environments. Reusing filesystem properties does not establish coverage of +these backends. On the local macOS ARM64 ASAN build, the embedded HDFS JVM needed +compressed oops/class pointers and CDS disabled to avoid a JVM address-space +allocation failure; this was a local BE configuration adjustment. + +## TPC-H / TPC-DS comparison + +The opt-in `test_deltalake_benchmark` suite compares native Delta and ordinary +Hive Parquet reads of **the same immutable Parquet objects**. It runs the existing +22 TPC-H SQL files and 103 TPC-DS SQL files (including the split query variants) +from `tools/`. TPC-H query 15 uses an equivalent CTE in place of `revenue0`, since +the external catalogs are read-only. TPC-H Q7 and TPC-DS Q85 fix join order for +both readers. Q7 follows the existing Hive regression: its default Hive plan +builds a supplier/customer intermediate before joining orders. Q85's default +Hive plan joins customer_demographics aliases on marital/education status before +joining web_returns. Both intermediates exceed the memory limit. JSON records mark +this control as `fixed_join_order=true`; report it separately from default-plan +measurements. TPC-DS uses the repository's `sf100` query +parameters on SF1 data; Q30 projects the 3.2 generator's `c_last_review_date` +string in place of the older query's `c_last_review_date_sk` column on both +readers. This is reader acceptance, not a certified TPC result or +a production throughput benchmark. + +Generate SF1 data with the existing `tools/tpch-tools/bin/build-tpch-dbgen.sh` +and `tools/tpcds-tools/bin/build-tpcds-tools.sh` tools in their supported Linux +environment. Run `dbgen -s 1` with `DSS_PATH` pointing to a dedicated output +directory, and `dsdgen -SCALE 1 -DIR ` from its tools directory. Retain +the generated `.tbl`/`.dat` files and tool distributions for reuse. The sources +used for local acceptance are the repository's TPC-H 3.0.0 and TPC-DS 3.2.0rc2 +archives, with SHA-256 respectively: + +- `30f0884c5b89fcddebf7daed264691a46d9e07049a3b89dfeb191b189c30cee3` +- `fa1d64cec6df6bd3ef50cff416172dd8a0ffc55500819343fe9af4dfd7ff36eb` + +Install `duckdb==1.4.4` in a Python environment (no DuckDB extensions required), +then convert the raw data: + +```bash +python3 docker/thirdparties/docker-compose/iceberg/scripts/deltalake/generate_benchmark_parquet.py \ + --tpch-generator-dir /path/to/tpch/dbgen --tpch-data-dir /path/to/tpch-data \ + --tpcds-generator-dir /path/to/tpcds/tools --tpcds-data-dir /path/to/tpcds-data \ + --output /path/to/benchmark +``` + +The converter uses the generators' DDL, preserves decimal/date types, explicitly +transcodes TPC-DS's Latin-1 text to UTF-8, and writes Snappy Parquet with 122,880-row +groups. It records row counts, sizes and SHA-256 in `manifest.json`. It refuses to +overwrite a nonempty output directory; reruns should reuse the retained fixtures +or choose a new directory. Equivalent Parquet encoding can produce different +bytes, so use the manifest corresponding to the files actually uploaded. + +Upload `delta_tpch/` and `delta_tpcds/` beneath `deltalakeWarehouse`, retaining the +layout `warehouse/delta_tpch/lineitem/data/part-00000.parquet` and its equivalents. +Copy the manifest into the Spark service's mounted `cache/deltalake/spark/benchmark/` +directory. In the optional Delta Compose environment, initialize metadata: + +```bash +docker compose --env-file iceberg.env -f iceberg.yaml --profile deltalake exec \ + spark-deltalake /opt/spark/bin/spark-submit \ + --properties-file /mnt/scripts/deltalake/spark-defaults.conf \ + /mnt/scripts/deltalake/prepare_benchmark.py \ + --manifest /delta-state/benchmark/manifest.json --warehouse s3a://delta/warehouse +``` + +The helper registers each Hive table at its `data/` subdirectory and runs +`CONVERT TO DELTA` on the parent directory, without rewriting Parquet. It checks +Hive's provider remains Parquet and Spark reads the expected row count through +both formats. Pointing a plain Hive table at the Delta root would also enumerate +log files, so the separate data directory is required. These fixtures have no +DV or concurrent writes; use the ordinary p0 suites for those semantics. + +Set `enableDeltaLakeTest=true` and `enableDeltaLakeBenchmarkTest=true` along with +the existing MinIO/HMS/warehouse settings, then run: + +```bash +./run-regression-test.sh --run -d external_table_p2/deltalake -s test_deltalake_benchmark +``` + +The suite computes the reference result through Hive and compares every Delta +and Hive execution against it. These are dynamic cross-reader comparisons, +not handwritten expected results. It uses the shared typed result comparator: +integer/decimal values remain exact, while floating aggregates use the framework's +relative tolerance (`1e-8` with a magnitude floor of one). It disables SQL/file caches, uses FileScannerV2 +and four pipeline tasks for both readers, records a first execution of each, then +three alternating measured repetitions. Each `DELTA_BENCHMARK` log record contains +the workload/query, result row count, first execution times and all six measured +times. Summarize per-query medians; investigate substantial absolute and relative +gaps using EXPLAIN/profiles. First executions are not guaranteed cold OS/object +store caches. Do not run fixture generation or exports concurrently with timing. + +Both readers use an 8-GiB query memory limit and a 120-second timeout. SQL failures +are logged as `DELTA_BENCHMARK_FAILURE`; the suite attempts the other reader and +remaining queries, then fails if any query failed. Failed queries have no valid +paired timing and must not be included in performance aggregates. Result mismatches +fail immediately. For a focused rerun, set `deltalakeBenchmarkQueryPattern` to a +regular expression matching `tpch/q7.sql` or `tpcds/query1.sql`, for example +`"tpch/q[789][.]sql"`. Its default `".*"` runs the full inventory. + +Delta M1 supplies basic versioned row/size statistics, collected asynchronously; +column NDV/histograms remain unavailable. For a steady-state comparison after +statistics are ready, optionally set `deltalakeBenchmarkStatisticsManifest` to the +absolute path of the retained converter's `manifest.json`. Before timing each +selected workload, the suite polls its tables together until every Delta EXPLAIN +cardinality matches the manifest. The wait allows one FE external row-count +cache refresh interval (`external_cache_refresh_time_minutes`, normally 10 minutes) +plus two minutes for asynchronous work. It logs those values and Hive's current +cardinality as `DELTA_BENCHMARK_STATISTICS`, without executing data queries. A +missing workload or statistics that fail to become ready fails the suite. When +unset, the original first-execution behavior is preserved. Record this warmup +setting in reports; neither mode establishes a cold OS/object-store cache. + +Doris can disable join reordering when row counts are unknown, which can yield +expensive intermediate results for multi-table queries. Apart from the documented +Q7/Q85 controls, the suite preserves optimizer-selected plans and reports failures. +Do not present a controlled or rewritten query as default-plan performance. + +Catalogs, HMS tables, Parquet files and Delta logs remain available after testing. +For repeat runs, invoke the regression directly; no data regeneration is needed. + +### Local TPC-DS comparison after basic statistics + +On 2026-11-02, all 103 TPC-DS query files passed in one official suite using the +retained SF1 data and production revision `cc12b56819`, which used the earlier +FE snapshot-statistics path. The 24 Delta cardinalities +were checked against the independent manifest before timing. The host, ASAN build, +shared settings, Q85 join-order control and Q30 adaptation match the earlier run. +Totals are sums of per-query medians from three alternating measured pairs: + +| Measurement | Queries | Hive (s) | Delta (s) | Delta / Hive | +| --- | ---: | ---: | ---: | ---: | +| Before statistics, 2026-10-31 | 103 | 232.545 | 293.214 | 1.2609 | +| After statistics, 2026-11-02 | 103 | 225.408 | 262.556 | 1.1648 | +| After, nonempty results only | 91 | 212.654 | 246.057 | 1.1571 | + +The overall difference narrows from 26.1% to 16.5%, but does not establish parity. +Separate profiles confirm smaller build sides for Q43 and Q72; Q22 still has a +large aggregation-related gap, and Q4/Q11/Q74 lost earlier Delta advantages. A +separate Q4 rerun confirmed a 44.2% gap and is excluded from the full-run aggregate. +These are local ASAN observations, not RELEASE/TPC certification or proof that +statistics alone explain every change. TPC-H was not rerun. See tracking issue +[#69884](https://github.com/apache/doris/issues/69884) for the evidence summary. + +## Planning scale + +This opt-in fixture declares one million active AddFiles in a classic checkpoint. It deliberately creates **no data objects**: it tests metadata COUNT and FE range planning, not Parquet read throughput or correctness across one million physical objects. The ordinary Delta p0 suites cover real data, deletes and schema evolution. + +Install PyArrow 21.0.0 in a Python environment. Use the `only-checkpoint-files` golden table from Delta v4.4.1 (commit `cebdc44dc07d331383e72811177af4ffc8638990`), renamed `only_checkpoint_files` by the fixture importer. Generate into a warehouse/database/table layout: + +```bash +python3 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/deltalake/generate_planning_fixture.py \ + --golden-table /path/to/golden/only_checkpoint_files \ + --output /path/to/warehouse/delta_bench/million_files +``` + +The source golden table is Apache-2.0 licensed. Its original source is [Delta's pinned golden tables](https://github.com/delta-io/delta/tree/cebdc44dc07d331383e72811177af4ffc8638990/connectors/golden-tables/src/main/resources/golden/only-checkpoint-files). The generator uses its checkpoint schema and protocol/metadata, replaces the table id, and writes synthetic AddFiles plus the checkpoint's own JSON commit. Older commits are intentionally absent. + +Run the connector's actual streaming source three times with a small heap: + +```bash +mvn -f fe/pom.xml -pl :fe-connector-deltalake -am install \ + -Dtest=DeltaLakePlanningScaleTest -DfailIfNoTests=false \ + -Dmaven.build.cache.enabled=false -DargLine=-Xmx512m \ + -Ddeltalake.planning.table=/path/to/warehouse/delta_bench/million_files +``` + +The test verifies every pass produced exactly 1,000,000 ranges and 527,000,000 declared bytes. It logs snapshot loading, the streaming decision, first-range latency, full enumeration time, and heap samples every 10,000 ranges. It is skipped unless the table property is supplied. The declared bytes are metadata, not measured data-file traffic. + +For Doris p2 acceptance, upload `delta_bench/million_files` beneath the configured `deltalakeWarehouse`, enable `enableDeltaLakeTest` and `enableDeltaLakePlanningTest`, and run: + +```bash +./run-regression-test.sh --run -d external_table_p2/deltalake -s test_deltalake_planning +``` + +This verifies cold/warm metadata COUNT and the streaming plan. Do not issue ordinary data queries against this fixture. Its absent Parquet objects also make an accidental fallback from metadata count fail visibly. + +## Local measurement + +Measured on 2026-10-30 with Java 17, Kernel 4.4.1, an 18-logical-CPU/48-GiB macOS host and a 512-MiB test JVM. The generated checkpoint was 7,685,296 bytes. These are individual runs, not a throughput benchmark or a cross-machine performance guarantee. + +| Pass | Snapshot (ms) | Streaming decision (ms) | First range (ms) | Full enumeration (ms) | Highest sampled heap (MiB) | +| --- | ---: | ---: | ---: | ---: | ---: | +| Cold | 1641.946 | 1.487 | 59.983 | 1863.624 | 199.9 | +| Warm 1 | 0.887 | 0.016 | 25.438 | 1645.469 | 111.0 | +| Warm 2 | 0.289 | 0.023 | 21.849 | 1458.675 | 112.9 | + +The cold snapshot includes first-use library initialization. Heap peaks are sampled, not continuous maxima. The streaming decision does not enumerate AddFiles. The warm runs reuse snapshot metadata while Kernel still enumerates the checkpoint. + +Against local MinIO, Doris metadata COUNT returned 1,000,000 in 1.648 and 1.372 seconds. Both profiles recorded zero data `FileReadBytes` and `FileReadCalls`. This count measurement is separate from the JVM range-enumeration test above. + +## Phase timing and cache decision + +`DeltaLakePlanningMetricsTest` uses the same opt-in property and heap command as the scale test. It times Kernel handler/iterator boundaries and expression evaluation, then creates the same range DTO shape used for this unpartitioned fixture. This probe is separate from the complete connector source measurement: it omits storage URI normalization, authentication-context entry and source buffering. Read/decode includes filesystem reads and Parquet decoding; replay/iteration is the remainder inside Kernel iteration. These are combined phase costs, not a claim to isolate disk wait from CPU time. + +| Probe | Checkpoint read/decode (ms) | Stats JSON parsing (ms) | Expression evaluation (ms) | Replay/iteration (ms) | Range DTO construction (ms) | Selected files | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| No filter | 310.370 | 0 | 2.118 | 226.255 | 1137.967 | 1,000,000 | +| `id < 0` (all files skipped) | 272.070 | 365.928 | 220.749 | 167.768 | 0 | 0 | + +For this local workload, M1 keeps `DefaultEngine` without adding a log/checkpoint content cache: snapshot loading already becomes sub-millisecond on a hit, while range construction and stats/replay work account for most enumeration time. A byte cache cannot remove those costs, and would need additional memory accounting and ownership rules. The production planner instead avoids copying an immutable range when the split covers the complete file, and avoids re-entering the authentication context when `hasNext()` already has a buffered range. Remote high-latency storage or a long JSON-only history may have a different balance and should be measured before choosing a content cache. + +An isolated repeat of the complete-source test after those two changes produced: + +| Pass | Snapshot (ms) | Streaming decision (ms) | First range (ms) | Full enumeration (ms) | Highest sampled heap (MiB) | +| --- | ---: | ---: | ---: | ---: | ---: | +| Cold | 1564.612 | 1.203 | 43.802 | 1529.518 | 169.7 | +| Warm 1 | 0.782 | 0.013 | 21.083 | 1472.836 | 132.7 | +| Warm 2 | 0.248 | 0.020 | 19.747 | 1418.931 | 131.6 | + +Both runs used the same fixture and 512-MiB heap. Individual timings and heap samples depend on JIT/GC state; this is evidence of preserved scale behavior and the observed improvement, not a statistical performance claim. diff --git a/regression-test/suites/external_table_p2/deltalake/test_deltalake_benchmark.groovy b/regression-test/suites/external_table_p2/deltalake/test_deltalake_benchmark.groovy new file mode 100644 index 00000000000000..a1d4d027d72660 --- /dev/null +++ b/regression-test/suites/external_table_p2/deltalake/test_deltalake_benchmark.groovy @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import org.apache.doris.regression.util.ResultUtils +import java.sql.SQLException + +// The reference is computed from Hive on the same physical Parquet files, not a constant result. +suite("test_deltalake_benchmark", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest")) + || !"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeBenchmarkTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String hmsPort = context.config.otherConfigs.get("deltalakeHmsPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + for (String reader : ["delta", "hive"]) { + sql "drop catalog if exists test_deltalake_benchmark_${reader}" + String metadata = reader == "delta" ? "'type'='deltalake', 'warehouse'='${warehouse}'" + : "'type'='hms', 'hive.metastore.uris'='thrift://${host}:${hmsPort}'" + sql """create catalog test_deltalake_benchmark_${reader} properties ( + ${metadata}, 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + } + sql "set enable_sql_cache=false" + sql "set enable_file_cache=false" + sql "set parallel_pipeline_task_num=4" + sql "set exec_mem_limit=8589934592" + sql "set query_timeout=120" + sql "set time_zone='UTC'" + String queryPattern = context.config.otherConfigs.getOrDefault("deltalakeBenchmarkQueryPattern", ".*") + String manifestPath = context.config.otherConfigs.getOrDefault("deltalakeBenchmarkStatisticsManifest", "") + Map statisticsManifest = manifestPath.isEmpty() ? [:] : new JsonSlurper().parse(new File(manifestPath)) + def cardinality = { String relation -> + String plan = sql("explain select * from ${relation}").collect { it[0] }.join("\n") + def matcher = plan =~ /cardinality=(\d+)/ + return matcher.find() ? matcher.group(1).toLong() : -1L + } + List failedQueries = [] + int executedQueries = 0 + File repository = new File(context.config.suitePath).canonicalFile.parentFile.parentFile + for (String workload : ["tpch", "tpcds"]) { + File directory = new File(repository, "tools/${workload}-tools/queries" + + (workload == "tpcds" ? "/sf100" : "")) + List queries = directory.listFiles().findAll { it.name.endsWith('.sql') }.sort { it.name } + assertEquals(workload == "tpch" ? 22 : 103, queries.size(), "${workload} query inventory") + if (!manifestPath.isEmpty() && queries.any { "${workload}/${it.name}" ==~ queryPattern }) { + // Optional steady-state measurement: wait for asynchronous Delta statistics before timing. + // The immutable fixture manifest is the oracle; never derive it from the tested connector. + Map tables = statisticsManifest[workload] + assertTrue(tables != null && !tables.isEmpty(), "Missing ${workload} statistics manifest") + def config = sql "admin show frontend config like 'external_cache_refresh_time_minutes'" + int refreshSeconds = (config[0][1] as int) * 60 + Map deltaRows = [:] + // Touch every table on each pass so their FE cache entries can refresh in the same interval. + awaitUntil(refreshSeconds + 120, 10) { + tables.each { table, fixture -> + deltaRows[table] = cardinality("test_deltalake_benchmark_delta.delta_${workload}.${table}") + } + tables.every { table, fixture -> deltaRows[table] == (fixture.rows as long) } + } + tables.sort().each { table, fixture -> + long expectedRows = fixture.rows as long + long hiveRows = cardinality("test_deltalake_benchmark_hive.delta_${workload}.${table}") + logger.info("DELTA_BENCHMARK_STATISTICS " + JsonOutput.toJson([ + workload: workload, table: table, expected_rows: expectedRows, + delta_cardinality: deltaRows[table], hive_cardinality: hiveRows])) + } + } + for (File file : queries) { + String queryName = "${workload}/${file.name}" + if (!(queryName ==~ queryPattern)) { + continue + } + executedQueries++ + // Apply the same control to both readers when Hive joins dimensions before facts + // and exceeds memory. Q7 also follows the existing Hive TPC-H regression. + boolean fixedJoinOrder = (workload == "tpch" && file.name == "q7.sql") + || (workload == "tpcds" && file.name == "query85.sql") + sql "set disable_join_reorder=${fixedJoinOrder}" + String query = file.getText("UTF-8") + if (workload == "tpcds" && file.name == "query30.sql") { + // TPC-DS 3.2's generator DDL omits the old query template's _sk suffix. + query = query.replace("c_last_review_date_sk", "c_last_review_date") + } + if (workload == "tpch" && file.name == "q15.sql") { + // The existing tool query refers to a view; make it a CTE for read-only catalogs. + query = """with revenue0 as ( + select l_suppkey supplier_no, sum(l_extendedprice * (1-l_discount)) total_revenue + from lineitem where l_shipdate >= date '1996-01-01' + and l_shipdate < date '1996-01-01' + interval '3' month + group by l_suppkey + ) ${query}""" + } + List reference = null + def runQuery = { String reader -> + sql "switch test_deltalake_benchmark_${reader}" + sql "use delta_${workload}" + long start = System.nanoTime() + List rows = sql(query) + double milliseconds = (System.nanoTime() - start) / 1_000_000.0 + List normalized = rows.collect { row -> + row.collect { value -> value instanceof BigDecimal ? value.stripTrailingZeros() : value } + }.sort { it.toString() } + if (reference == null) { + reference = normalized + } else { + // The shared typed comparator keeps exact decimals/integers and tolerates + // floating aggregate round-off (for example, TPC-DS Q39 stddev_samp). + ResultUtils.assertSparkDorisResultEquals(reference, normalized) + } + return milliseconds + } + // Try both readers even if one cannot execute a query. Keep every failure visible, + // continue collecting coverage, and fail the suite after the remaining queries run. + List failures = [] + def attempt = { String reader, String phase -> + try { + return runQuery(reader) + } catch (SQLException failure) { + Map detail = [workload: workload, query: file.name, reader: reader, + phase: phase, error: failure.message] + failures.add(detail) + logger.warn("DELTA_BENCHMARK_FAILURE " + JsonOutput.toJson(detail)) + return null + } + } + Map first = [hive: attempt("hive", "first"), delta: attempt("delta", "first")] + Map samples = [hive: [], delta: []] + for (int round = 0; round < 3 && failures.isEmpty(); round++) { + for (String reader : (round % 2 == 0 ? ["delta", "hive"] : ["hive", "delta"])) { + samples[reader].add(attempt(reader, "repeat-${round + 1}")) + } + } + if (!failures.isEmpty()) { + failedQueries.add(queryName) + } + logger.info("DELTA_BENCHMARK " + JsonOutput.toJson([ + workload: workload, query: file.name, rows: reference?.size(), + status: failures.isEmpty() ? "complete" : "failed", + fixed_join_order: fixedJoinOrder, + first_ms: first, samples_ms: samples])) + } + } + assertTrue(executedQueries > 0, "No benchmark queries matched ${queryPattern}") + assertTrue(failedQueries.isEmpty(), "Benchmark queries failed: ${failedQueries}") +} diff --git a/regression-test/suites/external_table_p2/deltalake/test_deltalake_planning.groovy b/regression-test/suites/external_table_p2/deltalake/test_deltalake_planning.groovy new file mode 100644 index 00000000000000..fe53c802b94ce7 --- /dev/null +++ b/regression-test/suites/external_table_p2/deltalake/test_deltalake_planning.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The optional fixture contains one million AddFile actions and deliberately no data objects. +// Use metadata COUNT only; ordinary data queries are not meaningful on this planning fixture. +suite("test_deltalake_planning", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest")) + || !"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakePlanningTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("deltalakeMinioPort") + String warehouse = context.config.otherConfigs.get("deltalakeWarehouse") + sql "drop catalog if exists test_deltalake_planning" + sql """create catalog test_deltalake_planning properties ( + 'type'='deltalake', 'warehouse'='${warehouse}', + 's3.endpoint'='http://${host}:${minioPort}', 's3.region'='us-east-1', + 's3.access_key'='admin', 's3.secret_key'='password', 'use_path_style'='true' + )""" + sql "set enable_count_push_down_for_external_table=true" + sql "set enable_external_table_batch_mode=true" + sql "set num_files_in_batch_mode=1" + sql "set enable_profile=true" + sql "set enable_sql_cache=false" + qt_cold_count "select count(*) from test_deltalake_planning.delta_bench.million_files" + qt_warm_count "select count(*) from test_deltalake_planning.delta_bench.million_files" + explain { + sql "select count(*) from test_deltalake_planning.delta_bench.million_files" + contains "Delta snapshot: 10" + // The streaming range carries its row count; the scan node has no table-level count yet. + contains "pushdown agg=COUNT" + } +} diff --git a/regression-test/suites/external_table_p2/deltalake/test_deltalake_storage.groovy b/regression-test/suites/external_table_p2/deltalake/test_deltalake_storage.groovy new file mode 100644 index 00000000000000..6977bdcd09f30b --- /dev/null +++ b/regression-test/suites/external_table_p2/deltalake/test_deltalake_storage.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper + +// Run the same results against one configured storage target at a time. +suite("test_deltalake_storage", "p2,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeTest")) + || !"true".equalsIgnoreCase(context.config.otherConfigs.get("enableDeltaLakeStorageTest"))) { + return + } + String propertiesPath = context.config.otherConfigs.get("deltalakeStoragePropertiesFile") + Map properties = new JsonSlurper().parse(new File(propertiesPath)) as Map + def quote = { value -> "'" + value.toString().replace('\\', '\\\\').replace("'", "\\'") + "'" } + String storageProperties = properties.collect { key, value -> "${quote(key)}=${quote(value)}" }.join(',') + sql "drop catalog if exists test_deltalake_storage" + sql """create catalog test_deltalake_storage properties ( + 'type'='deltalake', 'deltalake.catalog.type'='filesystem', ${storageProperties} + )""" + sql "set time_zone='UTC'" + sql "set enable_sql_cache=false" + sql "set enable_count_push_down_for_external_table=true" + for (boolean streaming : [false, true]) { + sql "set enable_external_table_batch_mode=${streaming}" + sql "set num_files_in_batch_mode=1" + // Small byte ranges also exercise file-global DV positions on this storage backend. + sql "set file_split_size=128" + "order_qt_${streaming}_mapping"("""select * from + test_deltalake_storage.delta_golden.table_with_columnmapping_mode_id""") + "order_qt_${streaming}_dv"("""select * from + test_deltalake_storage.delta_golden.dv_with_columnmapping""") + "qt_${streaming}_dv_count"("""select count(*) from + test_deltalake_storage.delta_golden.dv_with_columnmapping""") + "order_qt_${streaming}_partitioned_dv"("""select * from + test_deltalake_storage.delta_golden.dv_partitioned_with_checkpoint""") + "order_qt_${streaming}_ntz"("""select * from + test_deltalake_storage.delta_golden.data_reader_timestamp_ntz_name_mode""") + "order_qt_${streaming}_widening"("""select * from + test_deltalake_storage.delta_golden.type_widening_nested""") + "order_qt_${streaming}_checkpoint"("""select * from + test_deltalake_storage.delta_golden.v2_checkpoint_parquet""") + "order_qt_${streaming}_checkpoint_json"("""select * from + test_deltalake_storage.delta_golden.v2_checkpoint_json""") + } +} diff --git a/run-fe-ut.sh b/run-fe-ut.sh index dcb8c551c315bf..37c733be5fbb4a 100755 --- a/run-fe-ut.sh +++ b/run-fe-ut.sh @@ -200,6 +200,9 @@ FE_MODULES=("fe-common" "fe-core") # fe-connector-adbc is not upstream of fe-core, so -am cannot discover its tests # unless the module is named explicitly. FE_MODULES+=("fe-connector/fe-connector-adbc") +# Delta is a runtime plugin, not a fe-core test dependency. Include it explicitly so CI also +# runs its connector and metastore tests through -am. +FE_MODULES+=("fe-connector/fe-connector-deltalake") # The fluss connector. The other connectors' tests run because fe-core declares each one as a test # dependency (for Legacy413ProviderTypeContractTest) and -am pulls them into the reactor; fluss is # not among them, so it has to be named here or its tests never run.