From 4db6df61ed484da514acba761685c3f0b9f0b0b8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 10:59:46 +0800 Subject: [PATCH 1/2] [refactor](external) Remove Hive ACID, Avro and LakeSoul readers on master ### What problem does this PR solve? Related PR: #69312 Problem Summary: Port the removal of unsupported external readers to master. Adapt Hive transaction cleanup to the connector modules and preserve the unified reader initialization interfaces. Reject deprecated reads before file access or metadata-only count shortcuts, and remove obsolete fixtures and examples. Preserve Iceberg and Hudi Avro metadata dependencies. ### Release note Remove Hive transactional table reads, including insert-only tables, and standalone Avro file reads. LakeSoul remains unsupported. Iceberg and Hudi Avro metadata parsing is preserved. All Thrift definitions and IDs remain unchanged. ### Check List (For Author) - Test: Hive connector and HMS unit tests; targeted FE core unit tests; FE Checkstyle; clang-format 16; BE syntax checks; build hygiene; shell syntax and Thrift definition checks. - Behavior changed: Yes, reject unsupported external reads explicitly. - Does this need documentation: No, this ports the existing removal. (cherry picked from commit 518be1af40aa4de25d58819df4c393531f01c102, adapted to master) --- be/src/exec/operator/file_scan_operator.cpp | 11 +- be/src/exec/rowid_fetcher.cpp | 1 - be/src/exec/scan/file_scan_range_utils.h | 28 + be/src/exec/scan/file_scanner.cpp | 19 +- be/src/exec/scan/file_scanner_v2.cpp | 1 + be/src/format/orc/vorc_reader.cpp | 177 +------ be/src/format/orc/vorc_reader.h | 13 +- be/src/format/table/hive_reader.cpp | 1 - .../table/transactional_hive_common.cpp | 79 --- .../format/table/transactional_hive_common.h | 103 ---- .../table/transactional_hive_reader.cpp | 304 ----------- .../format/table/transactional_hive_reader.h | 91 ---- .../exec/scan/file_scan_range_utils_test.cpp | 25 + be/test/exec/scan/file_scanner_v2_test.cpp | 7 +- be/test/format/condition_cache_test.cpp | 19 - .../orc/orc_reader_init_column_test.cpp | 160 ------ build-for-release.sh | 2 +- .../create_preinstalled_scripts/run38.hql | 69 --- .../avro/avro_all_types/all_type.avro | Bin 1155 -> 0 bytes .../avro/avro_all_types/avro_all_types | Bin 666989 -> 0 bytes .../docker-compose/lakesoul/lakesoul.yaml.tpl | 64 --- .../docker-compose/lakesoul/meta_cleanup.sql | 11 - .../docker-compose/lakesoul/meta_init.sql | 144 ------ .../thirdparties/run-thirdparties-docker.sh | 47 +- fe/fe-connector/fe-connector-hive/pom.xml | 21 - .../doris/connector/hive/HiveAcidUtil.java | 486 ------------------ .../doris/connector/hive/HiveConnector.java | 8 +- .../connector/hive/HiveReadTransaction.java | 100 ---- .../hive/HiveReadTransactionManager.java | 62 --- .../connector/hive/HiveScanPlanProvider.java | 245 ++------- .../doris/connector/hive/HiveScanRange.java | 74 --- .../connector/hive/HiveAcidUtilTest.java | 263 ---------- .../hive/HiveFileListingCacheTest.java | 6 +- .../hive/HiveReadTransactionTest.java | 231 --------- .../connector/hive/HiveScanBatchModeTest.java | 128 +---- .../connector/hive/HiveScanRangeAcidTest.java | 106 ---- .../HiveScanRangePartitionValuesTest.java | 21 - .../hive/HiveUnsupportedReadsTest.java | 52 ++ .../doris/connector/hms/CachingHmsClient.java | 21 - .../doris/connector/hms/HmsAcidConstants.java | 39 -- .../apache/doris/connector/hms/HmsClient.java | 60 --- .../doris/connector/hms/ThriftHmsClient.java | 116 ----- .../hms/ThriftHmsClientWriteAcidTest.java | 137 ----- .../common/util/FileFormatConstants.java | 1 - .../fileformat/AvroFileFormatProperties.java | 49 -- .../fileformat/FileFormatProperties.java | 2 +- .../doris/datasource/scan/FileGroupInfo.java | 4 +- .../datasource/scan/FilePartitionUtils.java | 37 +- .../datasource/scan/FileQueryScanNode.java | 7 +- .../nereids/load/NereidsFileGroupInfo.java | 4 +- .../org/apache/doris/qe/SessionVariable.java | 7 - .../HttpStreamTableValuedFunction.java | 3 +- .../doris/common/util/BrokerUtilTest.java | 14 +- .../UnsupportedExternalReadsTest.java | 50 ++ .../AvroFileFormatPropertiesTest.java | 42 -- ...PluginDrivenScanNodeHiveStreamingTest.java | 3 +- gensrc/thrift/Descriptors.thrift | 5 +- gensrc/thrift/PlanNodes.thrift | 9 +- gensrc/thrift/Types.thrift | 2 +- regression-test/conf/regression-conf.groovy | 9 - .../test_hdfs_orc_group0_orc_files.out | 102 ---- .../external/conf/regression-conf.groovy | 3 - .../hive/test_hive_basic_type.groovy | 3 - .../test_hdfs_orc_group0_orc_files.groovy | 6 - samples/datalake/lakesoul/README.md | 277 ---------- samples/datalake/lakesoul/conf/my.conf | 28 - samples/datalake/lakesoul/conf/properties | 31 -- .../spark-conf/fairscheduler.xml.template | 31 -- .../spark-conf/log4j2.properties.template | 69 --- .../spark-conf/metrics.properties.template | 210 -------- .../data/spark-conf/spark-defaults.conf | 26 - .../spark-conf/spark-defaults.conf.template | 27 - .../data/spark-conf/spark-env.sh.template | 81 --- .../lakesoul/data/spark-conf/workers.template | 19 - samples/datalake/lakesoul/docker-compose.env | 25 - samples/datalake/lakesoul/docker-compose.yml | 214 -------- .../lakesoul/scripts/load_tpch_data.sh | 23 - .../datalake/lakesoul/scripts/start_doris.sh | 60 --- samples/datalake/lakesoul/sql/init_doris.sql | 13 - .../datalake/lakesoul/sql/meta_cleanup.sql | 11 - samples/datalake/lakesoul/sql/meta_init.sql | 145 ------ samples/datalake/lakesoul/sql/mysql_init.sql | 35 -- .../datalake/lakesoul/sql/prepare_data.sql | 41 -- samples/datalake/lakesoul/start_all.sh | 158 ------ .../datalake/lakesoul/start_doris_client.sh | 20 - .../datalake/lakesoul/start_flink_cdc_job.sh | 20 - .../datalake/lakesoul/start_flink_client.sh | 19 - .../datalake/lakesoul/start_mysql_client.sh | 20 - samples/datalake/lakesoul/start_spark_sql.sh | 19 - samples/datalake/lakesoul/stop_all.sh | 19 - .../lakesoul/work-dir/lakesoul.properties | 21 - 91 files changed, 264 insertions(+), 5312 deletions(-) delete mode 100644 be/src/format/table/transactional_hive_common.cpp delete mode 100644 be/src/format/table/transactional_hive_common.h delete mode 100644 be/src/format/table/transactional_hive_reader.cpp delete mode 100644 be/src/format/table/transactional_hive_reader.h delete mode 100755 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql delete mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro delete mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types delete mode 100644 docker/thirdparties/docker-compose/lakesoul/lakesoul.yaml.tpl delete mode 100644 docker/thirdparties/docker-compose/lakesoul/meta_cleanup.sql delete mode 100644 docker/thirdparties/docker-compose/lakesoul/meta_init.sql delete mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java create mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java delete mode 100644 fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java delete mode 100644 samples/datalake/lakesoul/README.md delete mode 100644 samples/datalake/lakesoul/conf/my.conf delete mode 100644 samples/datalake/lakesoul/conf/properties delete mode 100644 samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template delete mode 100644 samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template delete mode 100644 samples/datalake/lakesoul/data/spark-conf/metrics.properties.template delete mode 100755 samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf delete mode 100644 samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template delete mode 100755 samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template delete mode 100644 samples/datalake/lakesoul/data/spark-conf/workers.template delete mode 100644 samples/datalake/lakesoul/docker-compose.env delete mode 100644 samples/datalake/lakesoul/docker-compose.yml delete mode 100644 samples/datalake/lakesoul/scripts/load_tpch_data.sh delete mode 100644 samples/datalake/lakesoul/scripts/start_doris.sh delete mode 100644 samples/datalake/lakesoul/sql/init_doris.sql delete mode 100644 samples/datalake/lakesoul/sql/meta_cleanup.sql delete mode 100644 samples/datalake/lakesoul/sql/meta_init.sql delete mode 100644 samples/datalake/lakesoul/sql/mysql_init.sql delete mode 100644 samples/datalake/lakesoul/sql/prepare_data.sql delete mode 100644 samples/datalake/lakesoul/start_all.sh delete mode 100644 samples/datalake/lakesoul/start_doris_client.sh delete mode 100644 samples/datalake/lakesoul/start_flink_cdc_job.sh delete mode 100644 samples/datalake/lakesoul/start_flink_client.sh delete mode 100644 samples/datalake/lakesoul/start_mysql_client.sh delete mode 100644 samples/datalake/lakesoul/start_spark_sql.sh delete mode 100644 samples/datalake/lakesoul/stop_all.sh delete mode 100644 samples/datalake/lakesoul/work-dir/lakesoul.properties diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index e8666131ffbaca..7e6f2ad15791e7 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type_struct.h" #include "exec/operator/olap_scan_operator.h" #include "exec/operator/scan_operator.h" +#include "exec/scan/file_scan_range_utils.h" #include "exec/scan/file_scanner.h" #include "exec/scan/file_scanner_v2.h" #include "exec/scan/scanner_context.h" @@ -188,8 +189,8 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // ADBC only has a FileScannerV2 reader, and enable_file_scanner_v2 is a session variable marked // fuzzy=true, so the regression harness flips it to false at random. Without letting adbc // through unconditionally, those queries land in v1, which has no "adbc" branch, and come back - // with an undiagnosable NotSupported -- showing up in CI as a random failure. This is the same - // mechanism as is_transactional_hive below, pointed the other way. Loads are left alone: there + // with an undiagnosable NotSupported -- showing up in CI as a random failure. + // Loads are left alone: there // is no ADBC load path, so widening the rule to cover them would only route them somewhere they // still cannot run. // Fluss is in the same position, and reaches this the same way: FlussScanPlanProvider stamps the @@ -202,9 +203,6 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o scan_params.table_format_params.table_format_type == "fluss")) { return true; } - const bool is_transactional_hive = - scan_params.__isset.table_format_params && - scan_params.table_format_params.table_format_type == "transactional_hive"; // Hybrid scans advertise native Parquet at scan level because ranges can arrive after the // scanner is selected; FORMAT_JNI alone does not imply that every range uses JNI. const bool is_hybrid_native_parquet = scan_params.format_type == TFileFormatType::FORMAT_JNI && @@ -224,7 +222,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing // every file. return scanner_v2_requested && !is_load && - scan_params.format_type != TFileFormatType::FORMAT_LANCE && !is_transactional_hive; + scan_params.format_type != TFileFormatType::FORMAT_LANCE; } Status FileScanLocalState::_init_scanners(std::list* scanners) { @@ -247,6 +245,7 @@ Status FileScanLocalState::_init_scanners(std::list* scanners) { } else { scan_params = _split_source->get_params(); } + RETURN_IF_ERROR(validate_file_scan_support(*scan_params)); const bool is_load = state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr; // TODO: Use scanner v2 for all queries. diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 615b553990bfba..4f20f3e3dd8ddb 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -463,7 +463,6 @@ TFileRangeDesc RowIdStorageReader::build_external_fetch_range(const TFileRangeDe // needed by virtual columns, and do not mutate the FileMapping shared by other fetches. auto range = source_range; range.table_format_params.iceberg_params.__set_delete_files({}); - range.table_format_params.transactional_hive_params = TTransactionalHiveDesc {}; return range; } diff --git a/be/src/exec/scan/file_scan_range_utils.h b/be/src/exec/scan/file_scan_range_utils.h index cd4c06c953458b..9b9e91fc221441 100644 --- a/be/src/exec/scan/file_scan_range_utils.h +++ b/be/src/exec/scan/file_scan_range_utils.h @@ -17,10 +17,38 @@ #pragma once +#include "common/status.h" #include "gen_cpp/PlanNodes_types.h" namespace doris { +inline Status validate_file_scan_support(const TFileScanRangeParams& params, + const TFileRangeDesc* range = nullptr) { + // Deprecated table markers must never fall through to plain file reads: that would lose + // Hive transaction visibility and LakeSoul merge semantics, including in metadata counts. + auto validate_table = [](const TTableFormatFileDesc& table) -> Status { + if (table.table_format_type == "transactional_hive") { + return Status::NotSupported("Hive transactional table reads are no longer supported"); + } + if (table.table_format_type == "lakesoul") { + return Status::NotSupported("LakeSoul reads are no longer supported"); + } + return Status::OK(); + }; + if (params.__isset.table_format_params) { + RETURN_IF_ERROR(validate_table(params.table_format_params)); + } + if (range != nullptr && range->__isset.table_format_params) { + RETURN_IF_ERROR(validate_table(range->table_format_params)); + } + const auto format = range != nullptr && range->__isset.format_type ? range->format_type + : params.format_type; + if (format == TFileFormatType::FORMAT_AVRO) { + return Status::NotSupported("Avro file reads are no longer supported"); + } + return Status::OK(); +} + inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) { // Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a // missing data or delete file would silently return an incomplete or incorrect snapshot. diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 15ae816cad2586..ff210680576ece 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -80,7 +80,6 @@ #include "format/table/paimon_reader.h" #include "format/table/partition_column_filler.h" #include "format/table/remote_doris_reader.h" -#include "format/table/transactional_hive_reader.h" #include "format/table/trino_connector_jni_reader.h" #include "format/text/text_reader.h" #include "io/cache/block_file_cache_profile.h" @@ -152,6 +151,7 @@ void FileScanner::_configure_file_scan_handlers() { } Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { + RETURN_IF_ERROR(validate_file_scan_support(*_params)); RETURN_IF_ERROR(Scanner::init(state, conjuncts)); _get_block_timer = ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerGetBlockTime", 1); @@ -1030,6 +1030,7 @@ Status FileScanner::_get_next_reader() { } const TFileRangeDesc& range = _current_range; + RETURN_IF_ERROR(validate_file_scan_support(*_params, &range)); _current_range_path = range.path; if (!_partition_slot_index_map.empty()) { @@ -1456,20 +1457,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, octx.slot_id_to_filter_conjuncts = &_slot_id_to_filter_conjuncts; if (range.__isset.table_format_params && - range.table_format_params.table_format_type == "transactional_hive") { - // TransactionalHiveReader IS-A OrcReader, no wrapping needed - auto tran_orc_reader = TransactionalHiveReader::create_unique( - _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), - _io_ctx, file_meta_cache_ptr); - tran_orc_reader->set_create_row_id_column_iterator_func( - [this]() -> std::shared_ptr { - return _create_row_id_column_iterator(); - }); - init_status = static_cast(tran_orc_reader.get())->init_reader(&octx); - - _cur_reader = std::move(tran_orc_reader); - } else if (range.__isset.table_format_params && - range.table_format_params.table_format_type == "iceberg") { + range.table_format_params.table_format_type == "iceberg") { // IcebergOrcReader IS-A OrcReader (CRTP mixin), no wrapping needed std::unique_ptr iceberg_reader = IcebergOrcReader::create_unique( _kv_cache, _profile, _state, *_params, range, _state->batch_size(), @@ -1587,6 +1575,7 @@ Status FileScanner::_generate_truncate_columns(bool need_to_get_parsed_schema) { } Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) { + RETURN_IF_ERROR(validate_file_scan_support(*_params, &range)); _current_range = range; _file_cache_statistics.reset(new io::FileCacheStatistics()); diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 975f1a74e60517..b869533b38b641 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -376,6 +376,7 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile Status FileScannerV2::_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { + RETURN_IF_ERROR(validate_file_scan_support(params, &range)); if (!is_supported(params, range)) { return Status::NotSupported( "FileScannerV2 does not support table format {} with file format {}", diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index fb2c3578a34922..79d161902038c5 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -85,7 +85,6 @@ #include "format/orc/orc_file_reader.h" #include "format/table/iceberg_default_value.h" #include "format/table/iceberg_reader.h" -#include "format/table/transactional_hive_common.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" #include "orc/Exceptions.hh" @@ -619,7 +618,7 @@ Status OrcReader::init_schema_reader() { Status OrcReader::get_parsed_schema(std::vector* col_names, std::vector* col_types) { - const auto& root_type = _is_acid ? remove_acid(_reader->getType()) : _reader->getType(); + const auto& root_type = _reader->getType(); for (int i = 0; i < root_type.getSubtypeCount(); ++i) { col_names->emplace_back(root_type.getFieldName(i)); col_types->emplace_back(convert_to_doris_type(root_type.getSubtype(i))); @@ -653,22 +652,8 @@ Status OrcReader::_init_read_columns() { build_column_id_map(&root_type, 0); VLOG_DEBUG << "[OrcReader] Total mapped columns: " << _column_id_to_file_type.size(); - if (_is_acid) { - for (uint64_t i = 0; i < root_type.getSubtypeCount(); ++i) { - if (root_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) { - auto row_orc_type = root_type.getSubtype(i); - for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) { - _type_map.emplace(TransactionalHive::ROW + "." + row_orc_type->getFieldName(j), - row_orc_type->getSubtype(j)); - } - } else { - _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); - } - } - } else { - for (int i = 0; i < root_type.getSubtypeCount(); ++i) { - _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); - } + for (int i = 0; i < root_type.getSubtypeCount(); ++i) { + _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); } return Status::OK(); @@ -688,34 +673,6 @@ void OrcReader::_init_file_column_mapping() { } } -bool OrcReader::_check_acid_schema(const orc::Type& type) { - if (orc::TypeKind::STRUCT == type.getKind()) { - if (type.getSubtypeCount() != TransactionalHive::ACID_COLUMN_NAMES.size()) { - return false; - } - for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) { - const std::string& field_name = type.getFieldName(i); - std::string field_name_lower_case = field_name; - std::transform(field_name.begin(), field_name.end(), field_name_lower_case.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (field_name_lower_case != TransactionalHive::ACID_COLUMN_NAMES_LOWER_CASE[i]) { - return false; - } - } - } else { - return false; - } - return true; -} - -const orc::Type& OrcReader::remove_acid(const orc::Type& type) { - if (_check_acid_schema(type)) { - return *(type.getSubtype(TransactionalHive::ROW_OFFSET)); - } else { - return type; - } -} - // orc only support LONG, FLOAT, STRING, DATE, DECIMAL, TIMESTAMP, BOOLEAN to push down predicates static std::unordered_map TYPEKIND_TO_PREDICATE_TYPE = { {orc::TypeKind::BYTE, orc::PredicateDataType::LONG}, @@ -1328,25 +1285,12 @@ void OrcReader::_classify_columns_for_lazy_read( const std::unordered_map>& partition_columns, const std::unordered_map& missing_columns) { - if (_is_acid) { - _lazy_read_ctx.predicate_orc_columns.insert( - _lazy_read_ctx.predicate_orc_columns.end(), - TransactionalHive::READ_ROW_COLUMN_NAMES.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES.end()); - } - for (auto& read_table_col : _read_table_cols) { _lazy_read_ctx.all_read_columns.emplace_back(read_table_col); if (!predicate_table_columns.empty()) { auto iter = predicate_table_columns.find(read_table_col); if (iter == predicate_table_columns.end()) { - if (!_is_acid || - std::find(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end(), - read_table_col) == - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()) { - _lazy_read_ctx.lazy_read_columns.emplace_back(read_table_col); - } + _lazy_read_ctx.lazy_read_columns.emplace_back(read_table_col); } else { _lazy_read_ctx.predicate_columns.first.emplace_back(iter->first); _lazy_read_ctx.predicate_columns.second.emplace_back(iter->second.second); @@ -1478,44 +1422,14 @@ Status OrcReader::_init_orc_row_reader() { const auto& selected_type = _row_reader->getSelectedType(); int idx = 0; - if (_is_acid) { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - auto sub_type = selected_type.getSubtype(i); - if (sub_type->getKind() == orc::TypeKind::STRUCT) { - for (int j = 0; j < sub_type->getSubtypeCount(); ++j) { - _colname_to_idx[TransactionalHive::ROW + "." + sub_type->getFieldName(j)] = - idx++; - } - } else { - _colname_to_idx[selected_type.getFieldName(i)] = idx++; - } - } - } else { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - _colname_to_idx[selected_type.getFieldName(i)] = idx++; - } + for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { + _colname_to_idx[selected_type.getFieldName(i)] = idx++; } _type_map.clear(); - if (_is_acid) { - for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) { - if (selected_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) { - auto row_orc_type = selected_type.getSubtype(i); - for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) { - std::string field_name = - TransactionalHive::ROW + "." + row_orc_type->getFieldName(j); - _type_map.emplace(field_name, row_orc_type->getSubtype(j)); - } - } else { - std::string field_name = selected_type.getFieldName(i); - _type_map.emplace(field_name, selected_type.getSubtype(i)); - } - } - } else { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - std::string field_name = selected_type.getFieldName(i); - _type_map.emplace(field_name, selected_type.getSubtype(i)); - } + for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { + std::string field_name = selected_type.getFieldName(i); + _type_map.emplace(field_name, selected_type.getSubtype(i)); } _remaining_rows = _row_reader->getNumberOfRows(); @@ -2698,7 +2612,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo #endif { SCOPED_RAW_TIMER(&_statistics.predicate_filter_time); - _build_delete_row_filter(block, _batch->numElements); std::vector columns_to_filter; int column_to_keep = block->columns(); @@ -2717,9 +2630,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo filter_conjuncts.emplace_back(conjunct); } std::vector filters; - if (_delete_rows_filter_ptr) { - filters.push_back(_delete_rows_filter_ptr.get()); - } IColumn::Filter result_filter(block->rows(), 1); bool can_filter_all = false; RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( @@ -2753,12 +2663,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo Block::filter_block_internal(block, columns_to_filter, result_filter)); Block::erase_useless_column(block, column_to_keep); } else { - if (_delete_rows_filter_ptr) { - _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr, start_row); - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal( - block, columns_to_filter, (*_delete_rows_filter_ptr))); - } else if (_position_delete_ordered_rowids != nullptr || - (_deletion_vector != nullptr && !_deletion_vector->isEmpty())) { + if (_position_delete_ordered_rowids != nullptr || + (_deletion_vector != nullptr && !_deletion_vector->isEmpty())) { std::unique_ptr filter(new IColumn::Filter(block->rows(), 1)); _execute_filter_position_delete_rowids(*filter, start_row); RETURN_IF_CATCH_EXCEPTION( @@ -2795,52 +2701,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo void OrcReader::_fill_batch_vec(std::vector& result, orc::ColumnVectorBatch* batch, int idx) { - if (_is_acid) { - for (auto* field : dynamic_cast(batch)->fields) { - if (dynamic_cast(field) != nullptr) { - for (auto* row_field : dynamic_cast(field)->fields) { - result.push_back(row_field); - } - } else { - result.push_back(field); - } - } - } else { - for (auto* field : dynamic_cast(batch)->fields) { - result.push_back(field); - } - } -} - -void OrcReader::_build_delete_row_filter(const Block* block, size_t rows) { - // transactional hive orc delete row - if (_delete_rows != nullptr) { - _delete_rows_filter_ptr = std::make_unique(rows, 1); - auto* __restrict _pos_delete_filter_data = _delete_rows_filter_ptr->data(); - const auto& original_transaction_column = assert_cast(*remove_nullable( - block->get_by_position((*_col_name_to_block_idx) - [TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE]) - .column)); - const auto& bucket_id_column = assert_cast(*remove_nullable( - block->get_by_position( - (*_col_name_to_block_idx)[TransactionalHive::BUCKET_LOWER_CASE]) - .column)); - const auto& row_id_column = assert_cast(*remove_nullable( - block->get_by_position( - (*_col_name_to_block_idx)[TransactionalHive::ROW_ID_LOWER_CASE]) - .column)); - for (int i = 0; i < rows; ++i) { - auto original_transaction = original_transaction_column.get_int(i); - auto bucket_id = bucket_id_column.get_int(i); - auto row_id = row_id_column.get_int(i); - - AcidRowID transactional_row_id = {.original_transaction = original_transaction, - .bucket = bucket_id, - .row_id = row_id}; - if (_delete_rows->contains(transactional_row_id)) { - _pos_delete_filter_data[i] = 0; - } - } + for (auto* field : dynamic_cast(batch)->fields) { + result.push_back(field); } } @@ -2877,11 +2739,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s std::vector table_col_names; table_col_names.insert(table_col_names.end(), _lazy_read_ctx.predicate_columns.first.begin(), _lazy_read_ctx.predicate_columns.first.end()); - if (_is_acid) { - table_col_names.insert(table_col_names.end(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()); - } for (auto& table_col_name : table_col_names) { auto& column_with_type_and_name = block->get_by_position((*_col_name_to_block_idx)[table_col_name]); @@ -2921,9 +2778,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s column_guard.mutable_column()->resize(size); } - // transactional hive orc delete row - _build_delete_row_filter(block, size); - _filter = std::make_unique(size, 1); auto* __restrict result_filter_data = _filter->data(); bool can_filter_all = false; @@ -2937,9 +2791,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s filter_conjuncts.emplace_back(conjunct); } std::vector filters; - if (_delete_rows_filter_ptr) { - filters.push_back(_delete_rows_filter_ptr.get()); - } RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( filter_conjuncts, &filters, block, _filter.get(), &can_filter_all)); @@ -2954,7 +2805,7 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s _lazy_read_ctx.predicate_partition_columns.size() + _lazy_read_ctx.predicate_missing_columns.size()); for (auto& col : table_col_names) { - // clean block to read predicate columns and acid columns + // Clear the columns used to evaluate predicates. columns_to_clear.emplace_back((*_col_name_to_block_idx)[col]); } for (auto& col : _lazy_read_ctx.predicate_partition_columns) { diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 842ff89d4291ad..64ea2466aded19 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -43,7 +43,6 @@ #include "format/format_common.h" #include "format/table/table_format_reader.h" #include "format/table/table_schema_change_helper.h" -#include "format/table/transactional_hive_common.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" @@ -207,8 +206,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { _deletion_vector = deletion_vector; } - void set_delete_rows(const AcidRowIDSet* delete_rows) { _delete_rows = delete_rows; } - Status filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, void* arg); Status fill_dict_filter_column_names( @@ -253,7 +250,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { } return true; } - static const orc::Type& remove_acid(const orc::Type& type); bool count_read_rows() override { return true; } @@ -270,8 +266,7 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { bool has_delete_operations() const override { return (_position_delete_ordered_rowids != nullptr && !_position_delete_ordered_rowids->empty()) || - (_deletion_vector != nullptr && !_deletion_vector->isEmpty()) || - (_delete_rows != nullptr && !_delete_rows->empty()); + (_deletion_vector != nullptr && !_deletion_vector->isEmpty()); } // RowPositionProvider implementation @@ -370,8 +365,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { Status _init_read_columns(); void _init_file_column_mapping(); - static bool _check_acid_schema(const orc::Type& type); - // ---- set_fill_columns sub-functions ---- // Collect predicate columns from conjuncts for lazy materialization. void _collect_predicate_columns_from_conjuncts( @@ -414,7 +407,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { void _fill_batch_vec(std::vector& result, orc::ColumnVectorBatch* batch, int idx); - void _build_delete_row_filter(const Block* block, size_t rows); Status _get_next_block_impl(Block* block, size_t* read_rows, bool* eof); void _init_system_properties(); void _init_file_description(); @@ -826,7 +818,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { size_t _decimal_scale_params_index = 0; protected: - bool _is_acid = false; // Protected so Iceberg subclasses can register synthesized columns // in on_before_init_reader. LazyReadContext _lazy_read_ctx; @@ -836,8 +827,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { private: std::unique_ptr _filter; - const AcidRowIDSet* _delete_rows = nullptr; - std::unique_ptr _delete_rows_filter_ptr; VExprContextSPtrs _not_single_slot_filter_conjuncts; const std::unordered_map* _slot_id_to_filter_conjuncts = nullptr; diff --git a/be/src/format/table/hive_reader.cpp b/be/src/format/table/hive_reader.cpp index 5d24c587d55f6b..f540c8cb0d67f4 100644 --- a/be/src/format/table/hive_reader.cpp +++ b/be/src/format/table/hive_reader.cpp @@ -98,7 +98,6 @@ Status HiveOrcReader::on_before_init_reader(ReaderInitContext* ctx) { ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); - // _is_acid is false by default, no need to set explicitly return Status::OK(); } diff --git a/be/src/format/table/transactional_hive_common.cpp b/be/src/format/table/transactional_hive_common.cpp deleted file mode 100644 index 3d68a72c8dcba2..00000000000000 --- a/be/src/format/table/transactional_hive_common.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/transactional_hive_common.h" - -#include - -namespace doris { - -const std::string TransactionalHive::OPERATION = "operation"; -const std::string TransactionalHive::ORIGINAL_TRANSACTION = "originalTransaction"; -const std::string TransactionalHive::BUCKET = "bucket"; -const std::string TransactionalHive::ROW_ID = "rowId"; -const std::string TransactionalHive::CURRENT_TRANSACTION = "currentTransaction"; -const std::string TransactionalHive::ROW = "row"; - -const std::string TransactionalHive::OPERATION_LOWER_CASE = "operation"; -const std::string TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE = "originaltransaction"; -const std::string TransactionalHive::BUCKET_LOWER_CASE = "bucket"; -const std::string TransactionalHive::ROW_ID_LOWER_CASE = "rowid"; -const std::string TransactionalHive::CURRENT_TRANSACTION_LOWER_CASE = "currenttransaction"; -const std::string TransactionalHive::ROW_LOWER_CASE = "row"; - -const int TransactionalHive::ROW_OFFSET = 5; - -const std::vector TransactionalHive::DELETE_ROW_PARAMS = { - {TransactionalHive::ORIGINAL_TRANSACTION, - TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE, PrimitiveType::TYPE_BIGINT}, - {TransactionalHive::BUCKET, TransactionalHive::BUCKET_LOWER_CASE, PrimitiveType::TYPE_INT}, - {TransactionalHive::ROW_ID, TransactionalHive::ROW_ID_LOWER_CASE, - PrimitiveType::TYPE_BIGINT}}; -const std::vector TransactionalHive::READ_PARAMS = { - {TransactionalHive::ORIGINAL_TRANSACTION, - TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE, PrimitiveType::TYPE_BIGINT}, - {TransactionalHive::BUCKET, TransactionalHive::BUCKET_LOWER_CASE, PrimitiveType::TYPE_INT}, - {TransactionalHive::ROW_ID, TransactionalHive::ROW_ID_LOWER_CASE, - PrimitiveType::TYPE_BIGINT}}; -const std::vector TransactionalHive::DELETE_ROW_COLUMN_NAMES = { - DELETE_ROW_PARAMS[0].column_name, DELETE_ROW_PARAMS[1].column_name, - DELETE_ROW_PARAMS[2].column_name}; - -const std::vector TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE = { - DELETE_ROW_PARAMS[0].column_lower_case, DELETE_ROW_PARAMS[1].column_lower_case, - DELETE_ROW_PARAMS[2].column_lower_case}; - -const std::vector TransactionalHive::READ_ROW_COLUMN_NAMES = { - READ_PARAMS[0].column_name, READ_PARAMS[1].column_name, READ_PARAMS[2].column_name}; - -const std::vector TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE = { - READ_PARAMS[0].column_lower_case, READ_PARAMS[1].column_lower_case, - READ_PARAMS[2].column_lower_case}; - -const std::vector TransactionalHive::ACID_COLUMN_NAMES = { - OPERATION, ORIGINAL_TRANSACTION, BUCKET, ROW_ID, CURRENT_TRANSACTION, ROW}; - -const std::vector TransactionalHive::ACID_COLUMN_NAMES_LOWER_CASE = { - OPERATION_LOWER_CASE, ORIGINAL_TRANSACTION_LOWER_CASE, BUCKET_LOWER_CASE, - ROW_ID_LOWER_CASE, CURRENT_TRANSACTION_LOWER_CASE, ROW_LOWER_CASE}; - -const std::unordered_map TransactionalHive::DELETE_COL_NAME_TO_BLOCK_IDX = { - {DELETE_ROW_PARAMS[0].column_lower_case, 0}, - {DELETE_ROW_PARAMS[1].column_lower_case, 1}, - {DELETE_ROW_PARAMS[2].column_lower_case, 2}}; - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_common.h b/be/src/format/table/transactional_hive_common.h deleted file mode 100644 index 435294220fad24..00000000000000 --- a/be/src/format/table/transactional_hive_common.h +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "core/data_type/define_primitive_type.h" -#include "exec/common/hash_table/phmap_fwd_decl.h" - -namespace doris { -struct TransactionalHive { - static const std::string OPERATION; - static const std::string ORIGINAL_TRANSACTION; - static const std::string BUCKET; - static const std::string ROW_ID; - static const std::string CURRENT_TRANSACTION; - static const std::string ROW; - static const std::string OPERATION_LOWER_CASE; - static const std::string ORIGINAL_TRANSACTION_LOWER_CASE; - static const std::string BUCKET_LOWER_CASE; - static const std::string ROW_ID_LOWER_CASE; - static const std::string CURRENT_TRANSACTION_LOWER_CASE; - static const std::string ROW_LOWER_CASE; - static const int ROW_OFFSET; - struct Param { - const std::string column_name; - const std::string column_lower_case; - const PrimitiveType type; - }; - static const std::vector DELETE_ROW_PARAMS; - static const std::vector READ_PARAMS; - static const std::vector DELETE_ROW_COLUMN_NAMES; - static const std::vector DELETE_ROW_COLUMN_NAMES_LOWER_CASE; - static const std::vector READ_ROW_COLUMN_NAMES; - static const std::vector READ_ROW_COLUMN_NAMES_LOWER_CASE; - static const std::vector ACID_COLUMN_NAMES; - static const std::vector ACID_COLUMN_NAMES_LOWER_CASE; - - static const std::unordered_map DELETE_COL_NAME_TO_BLOCK_IDX; -}; - -// ACID row identifier for transactional Hive tables, used for delete row matching. -// Placed here (not in TransactionalHiveReader) to avoid circular dependency with OrcReader. -struct AcidRowID { - int64_t original_transaction; - int64_t bucket; - int64_t row_id; - - struct Hash { - size_t operator()(const AcidRowID& transactional_row_id) const { - size_t hash_value = 0; - hash_value ^= std::hash {}(transactional_row_id.original_transaction) + - 0x9e3779b9 + (hash_value << 6) + (hash_value >> 2); - hash_value ^= std::hash {}(transactional_row_id.bucket) + 0x9e3779b9 + - (hash_value << 6) + (hash_value >> 2); - hash_value ^= std::hash {}(transactional_row_id.row_id) + 0x9e3779b9 + - (hash_value << 6) + (hash_value >> 2); - return hash_value; - } - }; - - struct Eq { - bool operator()(const AcidRowID& lhs, const AcidRowID& rhs) const { - return lhs.original_transaction == rhs.original_transaction && - lhs.bucket == rhs.bucket && lhs.row_id == rhs.row_id; - } - }; -}; - -using AcidRowIDSet = flat_hash_set; - -inline bool operator<(const AcidRowID& lhs, const AcidRowID& rhs) { - if (lhs.original_transaction != rhs.original_transaction) { - return lhs.original_transaction < rhs.original_transaction; - } else if (lhs.bucket != rhs.bucket) { - return lhs.bucket < rhs.bucket; - } else if (lhs.row_id != rhs.row_id) { - return lhs.row_id < rhs.row_id; - } else { - return false; - } -} - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_reader.cpp b/be/src/format/table/transactional_hive_reader.cpp deleted file mode 100644 index cef77926f25445..00000000000000 --- a/be/src/format/table/transactional_hive_reader.cpp +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/transactional_hive_reader.h" - -#include - -#include - -#include "core/data_type/data_type_factory.hpp" -#include "format/orc/vorc_reader.h" -#include "format/table/table_schema_change_helper.h" -#include "transactional_hive_common.h" - -namespace doris { - -namespace io { -struct IOContext; -} // namespace io -class VExprContext; -} // namespace doris - -namespace doris { - -TransactionalHiveReader::TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, - const TFileRangeDesc& range, size_t batch_size, - const std::string& ctz, io::IOContext* io_ctx, - FileMetaCache* meta_cache) - : OrcReader(profile, state, params, range, batch_size, ctz, io_ctx, meta_cache, false) { - _init_transactional_hive_profile(); -} - -TransactionalHiveReader::TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, - const TFileRangeDesc& range, size_t batch_size, - const std::string& ctz, - std::shared_ptr io_ctx_holder, - FileMetaCache* meta_cache) - : OrcReader(profile, state, params, range, batch_size, ctz, std::move(io_ctx_holder), - meta_cache, false) { - _init_transactional_hive_profile(); -} - -void TransactionalHiveReader::_init_transactional_hive_profile() { - static const char* transactional_hive_profile = "TransactionalHiveProfile"; - ADD_TIMER(get_profile(), transactional_hive_profile); - _transactional_orc_profile.num_delete_files = ADD_CHILD_COUNTER( - get_profile(), "NumDeleteFiles", TUnit::UNIT, transactional_hive_profile); - _transactional_orc_profile.num_delete_rows = ADD_CHILD_COUNTER( - get_profile(), "NumDeleteRows", TUnit::UNIT, transactional_hive_profile); - _transactional_orc_profile.delete_files_read_time = - ADD_CHILD_TIMER(get_profile(), "DeleteFileReadTime", transactional_hive_profile); -} - -// ============================================================================ -// on_before_init_reader: ACID schema mapping -// ============================================================================ -Status TransactionalHiveReader::on_before_init_reader(ReaderInitContext* ctx) { - _column_descs = ctx->column_descs; - _fill_col_name_to_block_idx = ctx->col_name_to_block_idx; - RETURN_IF_ERROR(_extract_partition_values(*ctx->range, ctx->tuple_descriptor, - _fill_partition_values, - &_fill_partition_value_is_null)); - for (const auto& desc : *ctx->column_descs) { - if (desc.category == ColumnCategory::REGULAR || - desc.category == ColumnCategory::GENERATED) { - _col_names.push_back(desc.name); - } else if (desc.category == ColumnCategory::SYNTHESIZED && - desc.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { - auto topn_row_id_column_iter = _create_topn_row_id_column_iterator(); - this->register_synthesized_column_handler( - desc.name, - [iter = std::move(topn_row_id_column_iter), this, &desc]( - Block* block, size_t rows) -> Status { - return fill_topn_row_id(iter, desc.name, block, rows); - }); - continue; - } - } - - _is_acid = true; - // Add ACID column names (originalTransaction, bucket, rowId, etc.) - _col_names.insert(_col_names.end(), TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()); - ctx->column_names = _col_names; - - // Get ORC file type - const orc::Type* orc_type_ptr = nullptr; - RETURN_IF_ERROR(get_file_type(&orc_type_ptr)); - const auto& orc_type = *orc_type_ptr; - - // Add ACID metadata columns to table_info_node - for (auto idx = 0; idx < TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.size(); idx++) { - table_info_node_ptr->add_children(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE[idx], - TransactionalHive::READ_ROW_COLUMN_NAMES[idx], - std::make_shared()); - } - - // https://issues.apache.org/jira/browse/HIVE-15190 - auto row_orc_type = orc_type.getSubtype(TransactionalHive::ROW_OFFSET); - std::vector row_names; - std::map row_names_map; - for (uint64_t idx = 0; idx < row_orc_type->getSubtypeCount(); idx++) { - const auto& file_column_name = row_orc_type->getFieldName(idx); - row_names.emplace_back(file_column_name); - row_names_map.emplace(file_column_name, idx); - } - - // Match table columns to file columns by name - for (const auto& slot : ctx->tuple_descriptor->slots()) { - const auto& slot_name = slot->col_name(); - - if (std::count(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end(), slot_name) > 0) { - return Status::InternalError("Column {} conflicts with ACID metadata column", - slot_name); - } - - if (row_names_map.contains(slot_name)) { - std::shared_ptr child_node = nullptr; - RETURN_IF_ERROR(BuildTableInfoUtil::by_orc_name( - slot->type(), row_orc_type->getSubtype(row_names_map[slot_name]), child_node)); - auto file_column_name = fmt::format( - "{}.{}", TransactionalHive::ACID_COLUMN_NAMES[TransactionalHive::ROW_OFFSET], - slot_name); - table_info_node_ptr->add_children(slot_name, file_column_name, child_node); - } else { - table_info_node_ptr->add_not_exist_children(slot_name); - } - } - ctx->table_info_node = table_info_node_ptr; - return Status::OK(); -} - -// ============================================================================ -// on_after_init_reader: read delete delta files -// ============================================================================ -Status TransactionalHiveReader::on_after_init_reader(ReaderInitContext* /*ctx*/) { - std::string data_file_path = get_scan_range().path; - // the path in _range has the namenode prefix removed, - // and the file_path in delete file is full path, so we should add it back. - if (get_scan_params().__isset.hdfs_params && get_scan_params().hdfs_params.__isset.fs_name) { - std::string fs_name = get_scan_params().hdfs_params.fs_name; - if (!starts_with(data_file_path, fs_name)) { - data_file_path = fs_name + data_file_path; - } - } - - std::vector delete_file_col_names; - int64_t num_delete_rows = 0; - int64_t num_delete_files = 0; - std::filesystem::path file_path(data_file_path); - - // bucket_xxx_attemptId => bucket_xxx - auto remove_bucket_attemptId = [](const std::string& str) { - re2::RE2 pattern("^bucket_\\d+_\\d+$"); - if (re2::RE2::FullMatch(str, pattern)) { - size_t pos = str.rfind('_'); - if (pos != std::string::npos) { - return str.substr(0, pos); - } - } - return str; - }; - - SCOPED_TIMER(_transactional_orc_profile.delete_files_read_time); - for (const auto& delete_delta : - get_scan_range().table_format_params.transactional_hive_params.delete_deltas) { - const std::string file_name = file_path.filename().string(); - - std::vector delete_delta_file_names; - for (const auto& x : delete_delta.file_names) { - delete_delta_file_names.emplace_back(remove_bucket_attemptId(x)); - } - auto iter = std::find(delete_delta_file_names.begin(), delete_delta_file_names.end(), - remove_bucket_attemptId(file_name)); - if (iter == delete_delta_file_names.end()) { - continue; - } - auto delete_file = - fmt::format("{}/{}", delete_delta.directory_location, - delete_delta.file_names[iter - delete_delta_file_names.begin()]); - - TFileRangeDesc delete_range; - delete_range.__set_fs_name(get_scan_range().fs_name); - delete_range.path = delete_file; - delete_range.start_offset = 0; - delete_range.size = -1; - delete_range.file_size = -1; - - OrcReader delete_reader(get_profile(), get_state(), get_scan_params(), delete_range, - 256 /*batch_size*/, get_state()->timezone(), get_io_ctx(), - _meta_cache, false); - - auto acid_info_node = std::make_shared(); - for (auto idx = 0; idx < TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.size(); - idx++) { - auto const& table_column_name = - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE[idx]; - auto const& file_column_name = TransactionalHive::DELETE_ROW_COLUMN_NAMES[idx]; - acid_info_node->add_children(table_column_name, file_column_name, - std::make_shared()); - } - - OrcInitContext delete_ctx; - delete_ctx.column_names.assign( - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.end()); - delete_ctx.col_name_to_block_idx = const_cast*>( - &TransactionalHive::DELETE_COL_NAME_TO_BLOCK_IDX); - delete_ctx.table_info_node = acid_info_node; - RETURN_IF_ERROR(delete_reader.init_reader(&delete_ctx)); - - bool eof = false; - while (!eof) { - Block block; - for (const auto& i : TransactionalHive::DELETE_ROW_PARAMS) { - DataTypePtr data_type = DataTypeFactory::instance().create_data_type(i.type, false); - MutableColumnPtr data_column = data_type->create_column(); - block.insert(ColumnWithTypeAndName(std::move(data_column), data_type, - i.column_lower_case)); - } - eof = false; - size_t read_rows = 0; - RETURN_IF_ERROR(delete_reader.get_next_block(&block, &read_rows, &eof)); - if (read_rows > 0) { - static int ORIGINAL_TRANSACTION_INDEX = 0; - static int BUCKET_ID_INDEX = 1; - static int ROW_ID_INDEX = 2; - const auto& original_transaction_column = assert_cast( - *block.get_by_position(ORIGINAL_TRANSACTION_INDEX).column); - const auto& bucket_id_column = assert_cast( - *block.get_by_position(BUCKET_ID_INDEX).column); - const auto& row_id_column = assert_cast( - *block.get_by_position(ROW_ID_INDEX).column); - - DCHECK_EQ(original_transaction_column.size(), read_rows); - DCHECK_EQ(bucket_id_column.size(), read_rows); - DCHECK_EQ(row_id_column.size(), read_rows); - - for (int i = 0; i < read_rows; ++i) { - Int64 original_transaction = original_transaction_column.get_int(i); - Int64 bucket_id = bucket_id_column.get_int(i); - Int64 row_id = row_id_column.get_int(i); - AcidRowID delete_row_id = {original_transaction, bucket_id, row_id}; - _acid_delete_rows.insert(delete_row_id); - ++num_delete_rows; - } - } - } - ++num_delete_files; - } - if (num_delete_rows > 0) { - set_push_down_agg_type(TPushAggOp::NONE); - set_delete_rows(&_acid_delete_rows); - COUNTER_UPDATE(_transactional_orc_profile.num_delete_files, num_delete_files); - COUNTER_UPDATE(_transactional_orc_profile.num_delete_rows, num_delete_rows); - } - return Status::OK(); -} - -// ============================================================================ -// on_before_read_block: expand ACID columns into block -// TODO: Consider caching ACID column templates at init time to avoid repeated -// create_column + map update on every batch. Requires a block template mechanism. -// ============================================================================ -Status TransactionalHiveReader::on_before_read_block(Block* block) { - for (const auto& i : TransactionalHive::READ_PARAMS) { - DataTypePtr data_type = DataTypeFactory::instance().create_data_type(i.type, false); - MutableColumnPtr data_column = data_type->create_column(); - (*col_name_to_block_idx_ref())[i.column_lower_case] = block->columns(); - block->insert( - ColumnWithTypeAndName(std::move(data_column), data_type, i.column_lower_case)); - } - return Status::OK(); -} - -// ============================================================================ -// on_after_read_block: shrink ACID columns from block -// ============================================================================ -Status TransactionalHiveReader::on_after_read_block(Block* block, size_t* /*read_rows*/) { - Block::erase_useless_column(block, block->columns() - TransactionalHive::READ_PARAMS.size()); - for (const auto& i : TransactionalHive::READ_PARAMS) { - col_name_to_block_idx_ref()->erase(i.column_lower_case); - } - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_reader.h b/be/src/format/table/transactional_hive_reader.h deleted file mode 100644 index 1f8c26a7ff298f..00000000000000 --- a/be/src/format/table/transactional_hive_reader.h +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/factory_creator.h" -#include "common/status.h" -#include "format/orc/vorc_reader.h" -#include "format/table/table_schema_change_helper.h" -#include "format/table/transactional_hive_common.h" - -namespace doris { -class RuntimeState; -class TFileRangeDesc; -class TFileScanRangeParams; - -namespace io { -struct IOContext; -} // namespace io - -class Block; -class ShardedKVCache; -class VExprContext; - -// TransactionalHiveReader: directly inherits OrcReader (no composition wrapping). -// ACID column expansion/shrinking done via on_before_read_block/on_after_read_block hooks. -// Delete delta reading done via on_after_init_reader hook. -class TransactionalHiveReader final : public OrcReader, public TableSchemaChangeHelper { - ENABLE_FACTORY_CREATOR(TransactionalHiveReader); - -public: - TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, const TFileRangeDesc& range, - size_t batch_size, const std::string& ctz, io::IOContext* io_ctx, - FileMetaCache* meta_cache = nullptr); - TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, const TFileRangeDesc& range, - size_t batch_size, const std::string& ctz, - std::shared_ptr io_ctx_holder, - FileMetaCache* meta_cache = nullptr); - ~TransactionalHiveReader() final = default; - -protected: - // Hook: ACID schema mapping (add transactional columns, map row.* fields) - Status on_before_init_reader(ReaderInitContext* ctx) override; - - // Hook: read delete delta files - Status on_after_init_reader(ReaderInitContext* /*ctx*/) override; - - // Hook: expand ACID columns into block before reading - Status on_before_read_block(Block* block) override; - - // Hook: shrink ACID columns from block after reading - Status on_after_read_block(Block* block, size_t* read_rows) override; - -private: - void _init_transactional_hive_profile(); - - struct TransactionalHiveProfile { - RuntimeProfile::Counter* num_delete_files = nullptr; - RuntimeProfile::Counter* num_delete_rows = nullptr; - RuntimeProfile::Counter* delete_files_read_time = nullptr; - }; - - TransactionalHiveProfile _transactional_orc_profile; - AcidRowIDSet _acid_delete_rows; - std::vector _col_names; -}; - -} // namespace doris diff --git a/be/test/exec/scan/file_scan_range_utils_test.cpp b/be/test/exec/scan/file_scan_range_utils_test.cpp index bc8bbfaa465535..46e1cc7898a014 100644 --- a/be/test/exec/scan/file_scan_range_utils_test.cpp +++ b/be/test/exec/scan/file_scan_range_utils_test.cpp @@ -21,6 +21,31 @@ namespace doris { +TEST(FileScanRangeUtilsTest, RemovedFormatsCannotUseLegacyOrMetadataReads) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ORC); + EXPECT_TRUE(validate_file_scan_support(params, &range).ok()); + for (const auto* format : {"transactional_hive", "lakesoul"}) { + TTableFormatFileDesc table; + table.__set_table_format_type(format); + table.__set_table_level_row_count(10); + range.__set_table_format_params(table); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); + range.__isset.table_format_params = false; + params.__set_table_format_params(table); + EXPECT_FALSE(validate_file_scan_support(params).ok()); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); + params.__isset.table_format_params = false; + } + params.__set_format_type(TFileFormatType::FORMAT_AVRO); + EXPECT_FALSE(validate_file_scan_support(params).ok()); + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + range.__set_format_type(TFileFormatType::FORMAT_AVRO); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); +} + TEST(FileScanRangeUtilsTest, IcebergFilesCannotBeIgnored) { TFileRangeDesc range; range.table_format_params.__set_table_format_type("iceberg"); diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index c6c5c68051095d..3c13bb1a8342eb 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -495,11 +495,8 @@ TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesAndLoads) params.__set_format_type(TFileFormatType::FORMAT_ORC); TTableFormatFileDesc table_format_params; - table_format_params.__set_table_format_type("transactional_hive"); + table_format_params.__set_table_format_type("hive"); params.__set_table_format_params(table_format_params); - EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - - params.table_format_params.__set_table_format_type("hive"); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); query_options.__set_enable_file_scanner_v2(false); @@ -803,7 +800,7 @@ TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) { const auto unsupported = range_with_format("lakesoul", TFileFormatType::FORMAT_PARQUET); const auto status = FileScannerV2::TEST_validate_scan_range(params, unsupported); EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("lakesoul"), std::string::npos); + EXPECT_NE(status.to_string().find("LakeSoul"), std::string::npos); } // Scenario: external file scan IO contexts are query readers for SELECT so the shared file-cache diff --git a/be/test/format/condition_cache_test.cpp b/be/test/format/condition_cache_test.cpp index c0e9c86af64853..e8366fcc3dc776 100644 --- a/be/test/format/condition_cache_test.cpp +++ b/be/test/format/condition_cache_test.cpp @@ -27,7 +27,6 @@ #include "format/generic_reader.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" -#include "format/table/transactional_hive_common.h" namespace doris::vectorized { @@ -506,24 +505,6 @@ TEST_F(ConditionCacheDeleteOpsTest, OrcWithPositionDeletes_CacheSkipped) { EXPECT_EQ(cache, nullptr); } -// -- OrcReader: with ACID deletes -> cache skipped -- -TEST_F(ConditionCacheDeleteOpsTest, OrcWithAcidDeletes_CacheSkipped) { - TFileScanRangeParams params; - TFileRangeDesc range; - auto reader = OrcReader::create_unique(params, range, 4064, "", nullptr); - AcidRowIDSet acid_deletes; - acid_deletes.insert({1, 0, 5}); - reader->set_delete_rows(&acid_deletes); - - bool hit = false; - std::shared_ptr> cache; - std::shared_ptr ctx; - simulate_init_condition_cache(reader.get(), 99, "/data/file.orc", hit, cache, ctx); - - EXPECT_EQ(ctx, nullptr); - EXPECT_EQ(cache, nullptr); -} - // -- MockReader: with deletes (simulating Iceberg/Hive with inner deletes) -> cache skipped -- // In the new architecture, Iceberg readers inherit ParquetReader/OrcReader directly (CRTP), // so has_delete_operations() is resolved through the base reader. We use MockFileFormatReader diff --git a/be/test/format/orc/orc_reader_init_column_test.cpp b/be/test/format/orc/orc_reader_init_column_test.cpp index 9c685176196262..553d3046a60bef 100644 --- a/be/test/format/orc/orc_reader_init_column_test.cpp +++ b/be/test/format/orc/orc_reader_init_column_test.cpp @@ -68,164 +68,4 @@ TEST_F(OrcReaderInitColumnTest, InitReadColumn) { } } -TEST_F(OrcReaderInitColumnTest, CheckAcidSchemaTest) { - using namespace orc; - TFileScanRangeParams params; - TFileRangeDesc range; - auto _reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - // 1. Test standard ACID schema - { - // Create standard ACID structure - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("row", createStructType()); - - ASSERT_TRUE(_reader->_check_acid_schema(*acid_type)); - } - - // 2. Test case-insensitive field names - { - auto acid_type = createStructType(); - acid_type->addStructField("OPERATION", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("OriginalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("Bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("ROWID", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTRANSACTION", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("ROW", createStructType()); - - ASSERT_TRUE(_reader->_check_acid_schema(*acid_type)); - } - - // 3. Test non-ACID schema - field count mismatch - { - auto non_acid_type = createStructType(); - non_acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - non_acid_type->addStructField("originalTransaction", - createPrimitiveType(orc::TypeKind::LONG)); - // Only added two fields - - ASSERT_FALSE(_reader->_check_acid_schema(*non_acid_type)); - } - - // 4. Test non-ACID schema - field name mismatch - { - auto wrong_name_type = createStructType(); - wrong_name_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - wrong_name_type->addStructField("wrongName", createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - wrong_name_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("currentTransaction", - createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("row", createStructType()); - - ASSERT_FALSE(_reader->_check_acid_schema(*wrong_name_type)); - } - - // 5. Test non-struct type - { - auto int_type = createPrimitiveType(orc::TypeKind::INT); - ASSERT_FALSE(_reader->_check_acid_schema(*int_type)); - - auto string_type = createPrimitiveType(orc::TypeKind::STRING); - ASSERT_FALSE(_reader->_check_acid_schema(*string_type)); - } -} - -TEST_F(OrcReaderInitColumnTest, RemoveAcidTest) { - using namespace orc; - TFileScanRangeParams params; - TFileRangeDesc range; - auto _reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - // 1. Test removing ACID info from ACID schema - { - // Create ACID schema - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - - // Create actual data structure - auto row_type = createStructType(); - row_type->addStructField("id", createPrimitiveType(orc::TypeKind::INT)); - row_type->addStructField("name", createPrimitiveType(orc::TypeKind::STRING)); - acid_type->addStructField("row", std::move(row_type)); - - // Verify that after removing ACID we get the type of the row field - const orc::Type& removed_type = _reader->remove_acid(*acid_type); - ASSERT_EQ(removed_type.getKind(), orc::TypeKind::STRUCT); - ASSERT_EQ(removed_type.getSubtypeCount(), 2); // id and name fields - ASSERT_EQ(removed_type.getFieldName(0), "id"); - ASSERT_EQ(removed_type.getFieldName(1), "name"); - } - - // 2. Test that non-ACID schema remains unchanged - { - // Create normal schema - auto normal_type = createStructType(); - normal_type->addStructField("field1", createPrimitiveType(orc::TypeKind::INT)); - normal_type->addStructField("field2", createPrimitiveType(orc::TypeKind::STRING)); - - const orc::Type& result_type = _reader->remove_acid(*normal_type); - ASSERT_EQ(&result_type, normal_type.get()); // Should return the same type - ASSERT_EQ(result_type.getSubtypeCount(), 2); - ASSERT_EQ(result_type.getFieldName(0), "field1"); - ASSERT_EQ(result_type.getFieldName(1), "field2"); - } - - // 3. Test primitive types (non-struct) remain unchanged - { - auto int_type = createPrimitiveType(orc::TypeKind::INT); - const orc::Type& result_type = _reader->remove_acid(*int_type); - ASSERT_EQ(&result_type, int_type.get()); - ASSERT_EQ(result_type.getKind(), orc::TypeKind::INT); - } - - // 4. Test complex nested ACID schema - { - // Create nested ACID schema - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - - // Create complex row structure - auto row_type = createStructType(); - - // Add basic fields - row_type->addStructField("id", createPrimitiveType(orc::TypeKind::INT)); - - // Add array field - auto array_type = createListType(createPrimitiveType(orc::TypeKind::STRING)); - row_type->addStructField("tags", std::move(array_type)); - - // Add Map field - auto map_type = createMapType(createPrimitiveType(orc::TypeKind::STRING), - createPrimitiveType(orc::TypeKind::INT)); - row_type->addStructField("properties", std::move(map_type)); - - acid_type->addStructField("row", std::move(row_type)); - - // Verify structure after removing ACID - const orc::Type& removed_type = _reader->remove_acid(*acid_type); - ASSERT_EQ(removed_type.getKind(), orc::TypeKind::STRUCT); - ASSERT_EQ(removed_type.getSubtypeCount(), 3); // id, tags, properties - ASSERT_EQ(removed_type.getFieldName(0), "id"); - ASSERT_EQ(removed_type.getFieldName(1), "tags"); - ASSERT_EQ(removed_type.getFieldName(2), "properties"); - - // Verify field types - ASSERT_EQ(removed_type.getSubtype(0)->getKind(), orc::TypeKind::INT); - ASSERT_EQ(removed_type.getSubtype(1)->getKind(), orc::TypeKind::LIST); - ASSERT_EQ(removed_type.getSubtype(2)->getKind(), orc::TypeKind::MAP); - } -} - } // namespace doris diff --git a/build-for-release.sh b/build-for-release.sh index 5282a723e2938c..336ba30e95c88a 100755 --- a/build-for-release.sh +++ b/build-for-release.sh @@ -147,7 +147,7 @@ echo "CLOUD: ${OUTPUT_CLOUD}" sh build.sh --clean && USE_AVX2="${_USE_AVX2}" sh build.sh && - USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool --be-extension-ignore avro-scanner + USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool echo "Begin to pack" rm -rf "${OUTPUT}" diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql deleted file mode 100755 index 79b871822d1ea5..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql +++ /dev/null @@ -1,69 +0,0 @@ -drop table if exists `avro_all_types`; -create table `avro_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' -LOCATION - '/user/doris/preinstalled_data/avro/avro_all_types'; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro deleted file mode 100644 index c66017bb6243ee860863a938a7c08965f121c205..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1155 zcma)5O^ee&7@i43?3P_YkRp4~I3jq6LV|)6glt`x6%|=qPfHJzcIsv`NlHEz!jhwU z6;HCr{sxbNM`7^~c=Q)|^5DsfGm|u#v@3XM>GOWg^FGhp=k6iYdgLgM?j(}~8jv>| z$2iUB6!&pVCs91bCJsp;Fky^?nMc_)!TtT?IzEUZMoGw%Ri|483bQGC9Hvcy$AOxB z8ilW#EE7mg@34p{-7^W*g6$}C8Eqy^ORSa*)7VoVNlL7igj;b;@_I@w1gXMaN&~(* zmh9EWXJo$G70BQbVHrKQDSQ-W!D<^Vqm|}CE~-Yt5ECbP;6_YnkGD7+@Xtd$E>tEj z7M_(()GI7|nRvmR(YpE^Q)_1Bx{Z@=lUbE{(j*rGw+X%sy{NgAeH>=;s;c6#iKlc% zGM3`L$SA9EIi)?VQ>piBi(QA8ohvV^rI_%R`?q|<{~A%p+b`n!)bVLP0CX?i z-X8tAzUJH=O1q5GFJmZA*VqyNM1YWQ>1Xg0Jb?iDHoANZ0Hj@!oAbS%@PPodunhqK z-{9#-_@H~i`M!35*?sKvN+Ob!-d(>a2Ww&?(Eg8z?pv0A0(%3X`}TrJR}ioupTU3F JHh+0po4=>nXx{(; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types deleted file mode 100644 index 169ad69cb43a67c977f5b266708ec6680c91b5f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 666989 zcmeI%&ubLxx-am`L({erL{ORxgCoJ^z9;Fuj~h80;}MbIl?xXqa@diqm0qqK%n#GO z_CD(#_RyJ#1IHpb5*}E=TLzA>+wx66?`1$*E@BX#_;a>Z; zpZ$Je;YxdY{!(}8=g;)e+q!-5eiuJZU+H{pA2ZY4tsfm+K$YmPo@4nF+usMevvciE zcY1Dd3(cVC^0fT=5jptWvANkRS7tjq4D%0vC~pn$8$XY?2KYAj{7<+19UjG71N`Ps z;m<_)HVC_a0B?=(+dq51pPipx`en<1xWm=U=(e|_x#`6(J+ue^U(-IR?>|pp`8mDX zetLiI{}HnN{h)e5FQ3m_0uDy_(kuGkGsG8OAHL}rpL@#eI6(RM+6~ed9%DNWP(9{$ zGs5Q|cRLQydmQd&gfBc6cO0Vs*xbz+{RiZ)pS?Q{(*NKNfAv+D?2a#YU$|%oDFzqh z=4a@qe*flg-QJ(pyE*#p-^9K$)aU>D-c6kCUm|vjv&~D!ZU!H`H2mGSmcd!t?aRn+ z;%t2uZ1YXQPBdTpO~Tg&-R9-x8wTCGA#RuYjxdOaGMaT{8*++*;lHNsTfZRg@3iH{m%e%I{n1?Wge)AB61O38* zikRfdhoAbII_Rl|I4F)fqC%rKzZ4g3Y&(^;WqtjuuF^Tw}-z^ zqP3kw`6I`P3gWw-cu7=l81HS_S*YB73)6t%Z_Y*aYs%)M|Ma8D)(BtS=`Kmrj|usQ zo6e=_AG__Ps@^i@PFGugc7N__oBv$zWBI>RAN#X=?8^PKpUnS%H(wq*|6kmq|Jk0m zzDMo$?Cx+P%KsR>zkt(Xda0W(kxPG>?f$gW$=KC@zD@tlJ{@ZJ(x!1g4VQk&y7Ya! z`Y>@8rYx@1k|^ z>8bn2ADp~V#J=3|c=QG+hbdB@y@p)W%e6c!iaQh4vPjcNSr3(4b0?dMlqs)Qv2qow zd7i?RyXAbj=ts=;AZ1?J5;QmVQ|78j4fTkzh!nn+BInf2&ca-Kw9{RhoxdElqB^J| z?KhkQ9vdrq#ds05-o~^wAI$ao$L6ggXQrBu&o7?6xpFl=Tx;a^YD!&KC!oJv-`L}_ zBtU=w0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0!bhm-FBxN^=L$^b$23OJkQg}L<~g{Yj20* z+3R~NYv&?XqgHzvjg8Z{mt+6YHy5T3JsFJy>ytB&F1{LGj?ul758uCw*1@Nz?jL_} z@wJ*8=xGfNPYGia#1hW@~kNCOjOGvO(SJJRBp|kY${Twyk5o1RjlTD3Rmuy z^W~x+G1r5Xd1Xt`+}KZ{l5v|tUiFolmPa_jC6h*AP9g1hK@2#wz zi&%|X?PWAJPTyXR{YT$im^$=iG!Cp!&OEyKYIr$D_f9^1|1MewpPssZ{K3f^MeNHR zk4JBSa+o6Z*=xu}y@3W+M?2l6+4;*+E2@Jk(tg7^;IXlySBw`? z>upS1^TAxNe{9}5a%QUe`26D8n=4o2!?i|kucp*>bpra!^^HBQO9BK45FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAdm#2(QS9SQIAHnT6ZVn#q&IkOvF$WvG#T-p1r=evUVi+QuCvOz7FLyj1y#dN$ ziqvPXAs6*>EzgSL&P25=(lk=mL*>@o$)+M@%Ij6ET*Ydhr*P$NIbSaN5pz9AnOC+1 z&5iw(xhhgaJz^{(g>R+EId!wMFxMXKbeCr5FGsDY4ys7|4d;Nz#)@7sUPP_8F>TEU zbG`ntdF#lTspjMJi)U}HT#XOc8o9liQrFc9=r7kd_P8zy5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7dX5{O2(-RVX>8qsRqoro9D^E5IMLs7)q+o5>&`rgXgxro)M)m}zpJt95rGUOdm!$V3c95o>RU;@RtaD{JQ>R-;yX8I6t8x0hr8(Ki>S z4m}x-1M8DBk1oC%UXIbdlMmm&i`K!Xr|utraPmeG`*O$Q(Ho!~rbvDE8gfxD*Yd0= z?o3q6B26P@JydSZoop&nro3Lo%2ll9c?wtVmhY$3W-*66iY^>-N<3-eZ8`IW&FxTrJo41afnQA^h zzj*fM%GLOAt&!WSDRo_)fc|oQV~^{S009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXFB!Osj z+nsLIqYtHcsDOj{Qg9T$noaWHb(} zPtH8L_-c4LM)yuWeE%+52cMq0fBeD88%6BP9gjzEfO423_1SC4MZH|hv!b{&Q7wx! zjg<9JxixpPsYsdfdKD{Iv6|;8T)A7$my3SHTn|#_l`TPYV?SlCique#7>h{ZTPbo* z-RvyPwMRSMrP=w*Q7fv0D$;(#IpDFeqF0O;QR{6?Tl2wOuYYXbI&x;J`S|?e*_$g@ z1it zv|4v3;>GhkjZDN)6tVVpD4xB(x3YFFVl`^Dm(kcbeS10fAANIS>d=$XIIuoB^XTHM z;pG_JJNfYayJ#JJdg}i12Pbb7u`hQ#9=!p|VT#meuOS!paxKq_;?6|1EYdVm)Xx83PRJsQz!-JOUR z&+{}g5kpbL+S{Rc_WIt++PR3;sMTIZW8?Jg<=B7p&4sB$Pe$Xw`sB=`i?4>4V|4H2 z!}sr^b@1t_`^O)gyivry-0^ty1}KLqQlGtsT-3|8JS&Pj6Vh_Q$izLg^9)XmPqTzj!=ts`fqnvc&fp1rwpH9lNxE{8e{k|f5&Lq- owBK+Jcx 'CompactionCommit' then - select version - INTO rs_version - from partition_info - where table_id = NEW.table_id - and partition_desc = NEW.partition_desc - and version != NEW.version - and commit_op = 'CompactionCommit' - order by version desc - limit 1; - if rs_version >= 0 then - if NEW.version - rs_version >= 10 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - else - if NEW.version >= 10 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - end if; - RETURN NULL; - end if; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER partition_table_change - AFTER INSERT - ON partition_info - FOR EACH ROW -EXECUTE PROCEDURE partition_insert(); - -create table if not exists global_config -( - key text, - value text, - primary key (key) -); diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 0f83913380b2de..1c5ec1d5f83414 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -76,12 +76,12 @@ Usage: $0 --hive-modules comma separated hive modules to refresh All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio,fluss + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,kerberos,ranger,polaris,minio,fluss " exit 1 } DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,hudi,db2,oceanbase,kerberos,minio" -ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris,fluss" +ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,ranger,polaris,fluss" COMPONENTS=$2 HELP=0 STOP=0 @@ -294,8 +294,6 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_DB2=1 elif [[ "${element}"x == "oceanbase"x ]];then RUN_OCEANBASE=1 - elif [[ "${element}"x == "lakesoul"x ]]; then - RUN_LAKESOUL=1 elif [[ "${element}"x == "kerberos"x ]]; then RUN_KERBEROS=1 elif [[ "${element}"x == "minio"x ]]; then @@ -1542,43 +1540,6 @@ start_fluss() { compose_up_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" -d --wait } -start_lakesoul() { - echo "RUN_LAKESOUL" - cp "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml.tpl "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml - register_stack_metadata "lakesoul" "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" - compose_cmd "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" down --remove-orphans - sudo rm -rf "${ROOT}"/docker-compose/lakesoul/data - if [[ "${STOP}" -ne 1 ]]; then - echo "PREPARE_LAKESOUL_DATA" - compose_cmd "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" up -d - ## import tpch data into lakesoul - ## install rustup - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y - # shellcheck source=/dev/null - . "${HOME}/.cargo/env" - ## install rust nightly-2023-05-20 - rustup install nightly-2023-05-20 - ## download&generate tpch data - mkdir -p lakesoul/test_files/tpch/data - git clone https://github.com/databricks/tpch-dbgen.git - ( - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - mv *.tbl ../lakesoul/test_files/tpch/data - ) - export TPCH_DATA=$(realpath lakesoul/test_files/tpch/data) - ## import tpch data - git clone https://github.com/lakesoul-io/LakeSoul.git - # git checkout doris_dev - ( - cd LakeSoul/rust - cargo test load_tpch_data --package lakesoul-datafusion --features=ci -- --nocapture - ) - fi -} - dump_kerberos_container_state() { local container="$1" @@ -1982,10 +1943,6 @@ if [[ "${RUN_MARIADB}" -eq 1 ]]; then launch_component "mariadb" "${LOG_ROOT}/start_mariadb.log" start_mariadb fi -if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then - launch_component "lakesoul" "${LOG_ROOT}/start_lakesoul.log" start_lakesoul -fi - if [[ "${RUN_FLUSS}" -eq 1 ]]; then launch_component "fluss" "${LOG_ROOT}/start_fluss.log" start_fluss fi diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index 32ee285e2f7cca..7a50a02bf71037 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -128,27 +128,6 @@ under the License. test - - - ${project.groupId} - fe-filesystem-local - ${project.version} - test - - - - - commons-lang - commons-lang - test - ${project.groupId} fe-filesystem-hdfs-base diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java deleted file mode 100644 index 00b391ee3a18d8..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java +++ /dev/null @@ -1,486 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsAcidConstants; -import org.apache.doris.filesystem.FileEntry; -import org.apache.doris.filesystem.FileIterator; -import org.apache.doris.filesystem.FileSystem; -import org.apache.doris.filesystem.Location; - -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList.RangeResponse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Pure directory-name ACID state resolution for transactional Hive tables. - * - *

Plugin-side port of fe-core {@code AcidUtil.getAcidState}. It resolves, for one partition - * directory, the "best" base plus the working set of delta / delete-delta directories under a snapshot - * ({@code ValidTxnList} + {@code ValidWriteIdList}) and returns the surviving data files (base + - * non-delete deltas) plus the delete-delta descriptors. The BE later applies row deletes from the - * delete deltas.

- * - *

The fe-core version drags in {@code FileCacheValue}/{@code LocationPath}/{@code AcidInfo}/ - * {@code HivePartition}; those all drop out at the plugin boundary because the plugin lists files with the - * engine-injected Doris {@link FileSystem} and emits {@link HiveScanRange} directly. Only the pure name-parsing - * plus the {@code hive-common} {@code Valid*} algorithm ports.

- * - *

Ref: hive/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java#getAcidState (the fe-core copy - * exists because hive3 cannot read hive4 transaction tables and using hive4 directly is problematic).

- */ -public final class HiveAcidUtil { - private static final Logger LOG = LogManager.getLogger(HiveAcidUtil.class); - - private static final String HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX = "bucket_"; - private static final String DELTA_SIDE_FILE_SUFFIX = "_flush_length"; - - private HiveAcidUtil() { - } - - /** Resolved ACID state for one partition: surviving data files + delete-delta descriptors. */ - public static final class AcidState { - private final List dataFiles; - private final List deleteDeltas; - - AcidState(List dataFiles, List deleteDeltas) { - this.dataFiles = dataFiles; - this.deleteDeltas = deleteDeltas; - } - - /** Base + non-delete delta bucket files that survive the snapshot; each becomes a scan split. */ - public List getDataFiles() { - return dataFiles; - } - - /** Delete-delta directories (with the bucket file names inside) the BE must subtract. */ - public List getDeleteDeltas() { - return deleteDeltas; - } - } - - /** A single delete-delta directory plus the (filtered) delete file names inside it. */ - public static final class DeleteDelta { - private final String directoryLocation; - private final List fileNames; - - DeleteDelta(String directoryLocation, List fileNames) { - this.directoryLocation = directoryLocation; - this.fileNames = fileNames; - } - - public String getDirectoryLocation() { - return directoryLocation; - } - - public List getFileNames() { - return fileNames; - } - } - - private static final class ParsedBase { - private final long writeId; - private final long visibilityId; - - ParsedBase(long writeId, long visibilityId) { - this.writeId = writeId; - this.visibilityId = visibilityId; - } - } - - private static ParsedBase parseBase(String name) { - //format1 : base_writeId - //format2 : base_writeId_visibilityId detail: https://issues.apache.org/jira/browse/HIVE-20823 - name = name.substring("base_".length()); - int index = name.indexOf("_v"); - if (index == -1) { - return new ParsedBase(Long.parseLong(name), 0); - } - return new ParsedBase( - Long.parseLong(name.substring(0, index)), - Long.parseLong(name.substring(index + 2))); - } - - private static final class ParsedDelta implements Comparable { - private final long min; - private final long max; - private final String path; - private final int statementId; - private final boolean deleteDelta; - private final long visibilityId; - - ParsedDelta(long min, long max, String path, int statementId, - boolean deleteDelta, long visibilityId) { - this.min = min; - this.max = max; - this.path = path; - this.statementId = statementId; - this.deleteDelta = deleteDelta; - this.visibilityId = visibilityId; - } - - /* - * Smaller minWID orders first; - * If minWID is the same, larger maxWID orders first; - * Otherwise, sort by stmtID; files w/o stmtID orders first. - * - * Compactions (Major/Minor) merge deltas/bases but delete of old files - * happens in a different process; thus it's possible to have bases/deltas with - * overlapping writeId boundaries. The sort order helps figure out the "best" set of files - * to use to get data. - * This sorts "wider" delta before "narrower" i.e. delta_5_20 sorts before delta_5_10 (and delta_11_20) - */ - @Override - public int compareTo(ParsedDelta other) { - return min != other.min ? Long.compare(min, other.min) : - other.max != max ? Long.compare(other.max, max) : - statementId != other.statementId - ? Integer.compare(statementId, other.statementId) : - path.compareTo(other.path); - } - } - - private static boolean isValidMetaDataFile(FileSystem fileSystem, String baseDir) - throws IOException { - String fileLocation = baseDir + "_metadata_acid"; - try { - return fileSystem.exists(Location.of(fileLocation)); - } catch (IOException e) { - return false; - } - } - - private static boolean isValidBase(FileSystem fileSystem, String baseDir, - ParsedBase base, ValidWriteIdList writeIdList) throws IOException { - if (base.writeId == Long.MIN_VALUE) { - //Ref: https://issues.apache.org/jira/browse/HIVE-13369 - //such base is created by 1st compaction in case of non-acid to acid table conversion.(you - //will get dir: `base_-9223372036854775808`) - //By definition there are no open txns with id < 1. - //After this: https://issues.apache.org/jira/browse/HIVE-18192, txns(global transaction ID) => writeId. - return true; - } - - // hive 4 : just check "_v" suffix, before hive 4 : check `_metadata_acid` file in baseDir. - if ((base.visibilityId > 0) || isValidMetaDataFile(fileSystem, baseDir)) { - return writeIdList.isValidBase(base.writeId); - } - - // if here, it's a result of IOW - return writeIdList.isWriteIdValid(base.writeId); - } - - private static ParsedDelta parseDelta(String fileName, String deltaPrefix, String path) { - // format1: delta_min_max_statementId_visibilityId, delete_delta_min_max_statementId_visibilityId - // _visibilityId maybe not exists. - // detail: https://issues.apache.org/jira/browse/HIVE-20823 - // format2: delta_min_max_visibilityId, delete_delta_min_visibilityId - // when minor compaction runs, we collapse per statement delta files inside a single - // transaction so we no longer need a statementId in the file name - - long visibilityId = 0; - int visibilityIdx = fileName.indexOf("_v"); - if (visibilityIdx != -1) { - visibilityId = Long.parseLong(fileName.substring(visibilityIdx + 2)); - fileName = fileName.substring(0, visibilityIdx); - } - - boolean deleteDelta = deltaPrefix.equals("delete_delta_"); - - String rest = fileName.substring(deltaPrefix.length()); - int split = rest.indexOf('_'); - int split2 = rest.indexOf('_', split + 1); - long min = Long.parseLong(rest.substring(0, split)); - - if (split2 == -1) { - long max = Long.parseLong(rest.substring(split + 1)); - return new ParsedDelta(min, max, path, -1, deleteDelta, visibilityId); - } - - long max = Long.parseLong(rest.substring(split + 1, split2)); - int statementId = Integer.parseInt(rest.substring(split2 + 1)); - return new ParsedDelta(min, max, path, statementId, deleteDelta, visibilityId); - } - - private interface FileFilter { - boolean accept(String fileName); - } - - private static final class FullAcidFileFilter implements FileFilter { - @Override - public boolean accept(String fileName) { - return fileName.startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX) - && !fileName.endsWith(DELTA_SIDE_FILE_SUFFIX); - } - } - - private static final class InsertOnlyFileFilter implements FileFilter { - @Override - public boolean accept(String fileName) { - return true; - } - } - - /** - * Lists the immediate children (files and directories) of {@code dir}, non-recursively, via the - * engine-injected Doris {@link FileSystem#list} — the literal listing that mirrors the old - * {@code FileSystem.listStatus}. - * - *

Literal, not glob: uses {@code fs.list(loc)}, never {@code fs.listFiles(loc)}. The per-scheme - * filesystems ({@code DFSFileSystem}, {@code S3CompatibleFileSystem}) override {@code listFiles} with a - * glob-aware branch that would treat a location containing {@code [}/{@code *}/{@code ?} as a pattern; a hive - * partition/delta location can legitimately contain those characters, and the old {@code listStatus} never - * glob-expanded. Mirrors {@code HiveFileListingCache.listFromFileSystem}.

- */ - private static List listEntries(FileSystem fs, String dir) throws IOException { - List entries = new ArrayList<>(); - try (FileIterator it = fs.list(Location.of(dir))) { - while (it.hasNext()) { - entries.add(it.next()); - } - } - return entries; - } - - /** - * Lists the immediate file children of {@code dir} (directories excluded). - * - *

Mirrors fe-core {@code globList(fs, dir, false)} on a bare directory path: with no wildcard the - * glob has a {@code null} pattern, so it returns files only and skips any nested directory. The literal - * {@link #listEntries} returns both, so the directory entries are filtered out here.

- */ - private static List listFiles(FileSystem fs, String dir) throws IOException { - List files = new ArrayList<>(); - for (FileEntry entry : listEntries(fs, dir)) { - if (!entry.isDirectory()) { - files.add(entry); - } - } - return files; - } - - /** - * Resolves the ACID state of one partition directory under the given snapshot. - * - * @param fs engine-injected Doris file system for the partition location - * @param partitionPath the partition directory (e.g. {@code hdfs://.../data_id=200103}) - * @param txnValidIds the two serialized {@code Valid*} lists keyed by {@link HmsAcidConstants} - * @param isFullAcid full-ACID (bucket_ filter + delete deltas) vs insert-only (accept-all) - * @return surviving data files + delete-delta descriptors for the partition - */ - public static AcidState getAcidState(FileSystem fs, String partitionPath, - Map txnValidIds, boolean isFullAcid) throws IOException { - // Ref: https://issues.apache.org/jira/browse/HIVE-18192 - // Readers should use the combination of ValidTxnList and ValidWriteIdList(Table) for snapshot isolation. - // ValidReadTxnList implements ValidTxnList - // ValidReaderWriteIdList implements ValidWriteIdList - ValidTxnList validTxnList; - if (txnValidIds.containsKey(HmsAcidConstants.VALID_TXNS_KEY)) { - validTxnList = new ValidReadTxnList(); - validTxnList.readFromString(txnValidIds.get(HmsAcidConstants.VALID_TXNS_KEY)); - } else { - throw new RuntimeException("Miss ValidTxnList"); - } - - ValidWriteIdList validWriteIdList; - if (txnValidIds.containsKey(HmsAcidConstants.VALID_WRITEIDS_KEY)) { - validWriteIdList = new ValidReaderWriteIdList(); - validWriteIdList.readFromString(txnValidIds.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - } else { - throw new RuntimeException("Miss ValidWriteIdList"); - } - - //hdfs://xxxxx/user/hive/warehouse/username/data_id=200103 - // List all files and folders, without recursion. - List partitionEntries = listEntries(fs, partitionPath); - - String oldestBase = null; - long oldestBaseWriteId = Long.MAX_VALUE; - String bestBasePath = null; - long bestBaseWriteId = 0; - boolean haveOriginalFiles = false; - List workingDeltas = new ArrayList<>(); - - for (FileEntry entry : partitionEntries) { - if (entry.isDirectory()) { - String dirName = entry.name(); //dirName: base_xxx,delta_xxx,... - String dirPath = partitionPath + "/" + dirName; - - if (dirName.startsWith("base_")) { - ParsedBase base = parseBase(dirName); - if (!validTxnList.isTxnValid(base.visibilityId)) { - //checks visibilityTxnId to see if it is committed in current snapshot. - continue; - } - - long writeId = base.writeId; - if (oldestBaseWriteId > writeId) { - oldestBase = dirPath; - oldestBaseWriteId = writeId; - } - - if (((bestBasePath == null) || (bestBaseWriteId < writeId)) - && isValidBase(fs, dirPath, base, validWriteIdList)) { - //IOW will generator a base_N/ directory: https://issues.apache.org/jira/browse/HIVE-14988 - //So maybe need consider: https://issues.apache.org/jira/browse/HIVE-25777 - - bestBasePath = dirPath; - bestBaseWriteId = writeId; - } - } else if (dirName.startsWith("delta_") || dirName.startsWith("delete_delta_")) { - String deltaPrefix = dirName.startsWith("delta_") ? "delta_" : "delete_delta_"; - ParsedDelta delta = parseDelta(dirName, deltaPrefix, dirPath); - - if (!validTxnList.isTxnValid(delta.visibilityId)) { - continue; - } - - // No need check (validWriteIdList.isWriteIdRangeAborted(min,max) != RangeResponse.ALL) - // It is a subset of (validWriteIdList.isWriteIdRangeValid(min, max) != RangeResponse.NONE) - if (validWriteIdList.isWriteIdRangeValid(delta.min, delta.max) != RangeResponse.NONE) { - workingDeltas.add(delta); - } - } else { - //Sometimes hive will generate temporary directories(`.hive-staging_hive_xxx` ), - // which do not need to be read. - LOG.warn("Read Hive Acid Table ignore the contents of this folder:" + dirName); - } - } else { - haveOriginalFiles = true; - } - } - - if (bestBasePath == null && haveOriginalFiles) { - // ALTER TABLE nonAcidTbl SET TBLPROPERTIES ('transactional'='true'); - throw new UnsupportedOperationException("For no acid table convert to acid, please COMPACT 'major'."); - } - - if ((oldestBase != null) && (bestBasePath == null)) { - /* - * If here, it means there was a base_x (> 1 perhaps) but none were suitable for given - * {@link writeIdList}. Note that 'original' files are logically a base_Long.MIN_VALUE and thus - * cannot have any data for an open txn. We could check {@link deltas} has files to cover - * [1,n] w/o gaps but this would almost never happen... - * - * We only throw for base_x produced by Compactor since that base erases all history and - * cannot be used for a client that has a snapshot in which something inside this base is - * open. (Nor can we ignore this base of course) But base_x which is a result of IOW, - * contains all history so we treat it just like delta wrt visibility. Imagine, IOW which - * aborts. It creates a base_x, which can and should just be ignored.*/ - long[] exceptions = validWriteIdList.getInvalidWriteIds(); - String minOpenWriteId = ((exceptions != null) - && (exceptions.length > 0)) ? String.valueOf(exceptions[0]) : "x"; - throw new IOException( - String.format("Not enough history available for ({},{}). Oldest available base: {}", - validWriteIdList.getHighWatermark(), minOpenWriteId, oldestBase)); - } - - workingDeltas.sort(null); - - List deltas = new ArrayList<>(); - long current = bestBaseWriteId; - int lastStatementId = -1; - ParsedDelta prev = null; - // find need read delta/delete_delta file. - for (ParsedDelta next : workingDeltas) { - if (next.max > current) { - if (validWriteIdList.isWriteIdRangeValid(current + 1, next.max) != RangeResponse.NONE) { - deltas.add(next); - current = next.max; - lastStatementId = next.statementId; - prev = next; - } - } else if ((next.max == current) && (lastStatementId >= 0)) { - //make sure to get all deltas within a single transaction; multi-statement txn - //generate multiple delta files with the same txnId range - //of course, if maxWriteId has already been minor compacted, - // all per statement deltas are obsolete - - deltas.add(next); - prev = next; - } else if ((prev != null) - && (next.max == prev.max) - && (next.min == prev.min) - && (next.statementId == prev.statementId)) { - // The 'next' parsedDelta may have everything equal to the 'prev' parsedDelta, except - // the path. This may happen when we have split update and we have two types of delta - // directories- 'delta_x_y' and 'delete_delta_x_y' for the SAME txn range. - - // Also note that any delete_deltas in between a given delta_x_y range would be made - // obsolete. For example, a delta_30_50 would make delete_delta_40_40 obsolete. - // This is valid because minor compaction always compacts the normal deltas and the delete - // deltas for the same range. That is, if we had 3 directories, delta_30_30, - // delete_delta_40_40 and delta_50_50, then running minor compaction would produce - // delta_30_50 and delete_delta_30_50. - deltas.add(next); - prev = next; - } - } - - List dataFiles = new ArrayList<>(); - List deleteDeltas = new ArrayList<>(); - - FileFilter fileFilter = isFullAcid ? new FullAcidFileFilter() : new InsertOnlyFileFilter(); - - // delta directories - for (ParsedDelta delta : deltas) { - String location = delta.path; - List entries = listFiles(fs, location); - if (delta.deleteDelta) { - List deleteDeltaFileNames = new ArrayList<>(); - for (FileEntry entry : entries) { - String name = entry.name(); - if (fileFilter.accept(name)) { - deleteDeltaFileNames.add(name); - } - } - deleteDeltas.add(new DeleteDelta(location, deleteDeltaFileNames)); - continue; - } - for (FileEntry entry : entries) { - if (fileFilter.accept(entry.name())) { - dataFiles.add(entry); - } - } - } - - // base - if (bestBasePath != null) { - List entries = listFiles(fs, bestBasePath); - for (FileEntry entry : entries) { - if (fileFilter.accept(entry.name())) { - dataFiles.add(entry); - } - } - } - - if (!isFullAcid && !deleteDeltas.isEmpty()) { - throw new RuntimeException("No Hive Full Acid Table have delete_delta_* Dir."); - } - return new AcidState(dataFiles, deleteDeltas); - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 7b6c5656806bfe..013db6a79f0522 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -96,12 +96,8 @@ public class HiveConnector implements Connector { private volatile HadoopAuthenticator pluginAuth; private volatile boolean pluginAuthComputed; - // Read-transaction manager for transactional (ACID) Hive scans. One per connector, keyed by query. - // Plugin-owned; live since the read cutover wired its query-finish commit (see the manager). - private final HiveReadTransactionManager readTxnManager = new HiveReadTransactionManager(); - // Connector-owned directory-listing cache, shared by the (per-call) scan provider and metadata. One per - // connector (like readTxnManager above) — the scan provider / metadata are rebuilt per query, so the cache + // connector — the scan provider / metadata are rebuilt per query, so the cache // must live on the long-lived connector to survive across scans. Built from catalog props. Its // metastore-metadata sibling is the CachingHmsClient wrapping the HmsClient. private final HiveFileListingCache fileListingCache; @@ -256,7 +252,7 @@ private Connector resolveSiblingOwner(ConnectorTableHandle handle) { @Override public ConnectorScanPlanProvider getScanPlanProvider() { - return new HiveScanPlanProvider(getOrCreateClient(), props, context, readTxnManager, fileListingCache); + return new HiveScanPlanProvider(getOrCreateClient(), props, context, fileListingCache); } /** diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java deleted file mode 100644 index 2c33edb961e13a..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsClient; -import org.apache.doris.connector.spi.DorisConnectorException; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Holds the state of one Hive read transaction, used when reading a transactional (ACID) Hive table. - * Each instance is bound to a single query. - * - *

Plugin-side port of fe-core {@code HiveTransaction}. It drives the four read-side metastore - * primitives through the shared {@link HmsClient} ({@code openTxn} / {@code acquireSharedLock} / - * {@code getValidWriteIds} / {@code commitTxn}) instead of the fe-core {@code HMSExternalCatalog}/ - * {@code HMSCachedClient}/{@code TableNameInfo} chain — the table identity is carried as plain - * {@code (dbName, tableName)} strings.

- */ -public class HiveReadTransaction { - private final String queryId; - private final String user; - private final String dbName; - private final String tableName; - private final boolean isFullAcid; - private final HmsClient hmsClient; - - private long txnId; - private final List partitionNames = new ArrayList<>(); - - private Map txnValidIds = null; - - public HiveReadTransaction(String queryId, String user, String dbName, String tableName, - boolean isFullAcid, HmsClient hmsClient) { - this.queryId = queryId; - this.user = user; - this.dbName = dbName; - this.tableName = tableName; - this.isFullAcid = isFullAcid; - this.hmsClient = hmsClient; - } - - public String getQueryId() { - return queryId; - } - - public void addPartition(String partitionName) { - this.partitionNames.add(partitionName); - } - - public boolean isFullAcid() { - return isFullAcid; - } - - /** - * Acquires a shared read lock and fetches the snapshot ({@code ValidTxnList} + {@code - * ValidWriteIdList}) for the current transaction, memoizing so the lock is taken at most once. The - * lock is released when the transaction is committed at query finish. - */ - public Map getValidWriteIds() { - if (txnValidIds == null) { - hmsClient.acquireSharedLock(queryId, txnId, user, dbName, tableName, partitionNames, 5000); - txnValidIds = hmsClient.getValidWriteIds(dbName + "." + tableName, txnId); - } - return txnValidIds; - } - - public void begin() { - try { - this.txnId = hmsClient.openTxn(user); - } catch (RuntimeException e) { - throw new DorisConnectorException(e.getMessage(), e); - } - } - - public void commit() { - try { - hmsClient.commitTxn(txnId); - } catch (RuntimeException e) { - throw new DorisConnectorException(e.getMessage(), e); - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java deleted file mode 100644 index 8ad06a6a776eb8..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Manages the read transaction of each query against transactional (ACID) Hive tables. For each query a - * {@link HiveReadTransaction} is registered (which opens the metastore transaction); when the query - * finishes it is deregistered (which commits the transaction, releasing the shared read lock). - * - *

Plugin-side port of fe-core {@code HiveTransactionMgr}. Plugin-owned and live since the read - * cutover: a transactional-hive scan registers via {@code HiveScanPlanProvider.planAcidScan}, and the - * fe-core query-finish callback ({@code PluginDrivenScanNode}) drives {@link #deregister} through - * {@code releaseReadTransaction}. The legacy fe-core {@code HiveScanNode} / - * {@code Env.getCurrentHiveTransactionMgr()} are removed.

- */ -public class HiveReadTransactionManager { - private static final Logger LOG = LogManager.getLogger(HiveReadTransactionManager.class); - - private final Map txnMap = new ConcurrentHashMap<>(); - - /** - * Opens the transaction (see {@link HiveReadTransaction#begin}) and registers it under its query id, so - * the query-finish path can find and commit it. Mirrors fe-core: one {@code HiveReadTransaction} is - * created per transactional-table scan (each pins its own table's write-id snapshot), keyed by query id. - */ - public void register(HiveReadTransaction txn) { - txn.begin(); - txnMap.put(txn.getQueryId(), txn); - } - - public void deregister(String queryId) { - HiveReadTransaction txn = txnMap.remove(queryId); - if (txn != null) { - try { - txn.commit(); - } catch (RuntimeException e) { - LOG.warn("failed to commit hive read txn: " + queryId, e); - } - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java index 18c4b142adc4ef..94b8edd86d306e 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java @@ -35,18 +35,14 @@ import org.apache.doris.connector.spi.scan.ConnectorScanRequest; import org.apache.doris.connector.spi.scan.ConnectorSplitSource; import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; -import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileSystem; import org.apache.doris.foundation.format.FileCompression; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileScanRangeParams; -import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -61,7 +57,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.UnaryOperator; /** * Scan plan provider for Hive tables. @@ -74,8 +69,7 @@ *
  • Return {@link HiveScanRange} for each file split
  • * * - *

    Non-transactional tables can stream file splits without materializing a partition's ranges. - * Transactional tables retain snapshot-aware synchronous planning.

    + *

    Tables can stream file splits without materializing a partition's ranges.

    */ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { @@ -89,12 +83,6 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { private static final int MAX_PARTITIONS = 100000; private static final int DEFAULT_BATCH_THRESHOLD = 1024; - /** - * Connector-internal signal (consumed by {@link #populateScanLevelParams}) marking a full-ACID - * (transactional) Hive scan; drives the scan-level {@code table_format_type} stamp. Not a BE-facing key. - */ - public static final String PROP_TRANSACTIONAL_HIVE = "transactional_hive"; - /** * Session variable gating the OpenX-JSON "read the whole JSON row into one CSV column" mode (legacy * SessionVariable.read_hive_json_in_one_column). Byte-identical to the fe-core session-var name; it is @@ -103,29 +91,22 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { private static final String SESSION_READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column"; static final String SCAN_REUSE_NAMESPACE = "hms.scan-reuse"; - /** Input format of a full-ACID (ORC) transactional Hive table; other formats are rejected. */ - private static final String ORC_ACID_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; - private final HmsClient hmsClient; private final HiveCatalogProperties catalogProperties; - // Engine-owned, per-catalog filesystem accessor. The non-ACID listing path borrows the Doris FileSystem via + // Engine-owned, per-catalog filesystem accessor. The listing path borrows the Doris FileSystem via // storage().getFileSystem(session) (never closes it — the engine owns its lifecycle) to list partition // directories, replacing bare Hadoop FileSystem.get (FIX-HIVEFS: the hive plugin bundles no HDFS impl). private final ConnectorContext context; - private final HiveReadTransactionManager readTxnManager; // Connector-owned directory-listing cache (the SAME instance HiveConnector shares with HiveConnectorMetadata), - // so a repeated scan of the same partition directory is served from the cache instead of re-listing. Only the - // plain (non-ACID) path uses it; the ACID path lists via HiveAcidUtil and is uncached (legacy parity). + // so a repeated scan of the same partition directory is served from the cache instead of re-listing. private final HiveFileListingCache fileListingCache; private final PartitionBatchProfile partitionBatchProfile = new PartitionBatchProfile(); public HiveScanPlanProvider(HmsClient hmsClient, HiveCatalogProperties catalogProperties, - ConnectorContext context, HiveReadTransactionManager readTxnManager, - HiveFileListingCache fileListingCache) { + ConnectorContext context, HiveFileListingCache fileListingCache) { this.hmsClient = hmsClient; this.catalogProperties = catalogProperties; this.context = context; - this.readTxnManager = readTxnManager; this.fileListingCache = fileListingCache; } @@ -143,14 +124,10 @@ public boolean usesHiveParquetInt96TimeZone() { @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); + rejectTransactionalRead(hiveHandle); if (session == null || !session.isExternalScanTaskReuseEnabled()) { return doPlanScan(session, request); } - if (hiveHandle.isTransactional()) { - // ACID / INSERT_ONLY reads open a per-scan read transaction with a write-id snapshot and - // a shared metastore lock; reusing the planned ranges would skip that transaction. - return doPlanScan(session, request); - } // Statement-scoped reuse: within one statement the identical scan (same table, same // partition set, same formats) plans once and every duplicated relation shares the result. // The scope is NONE for offline planning and tests, in which case the loader runs on every @@ -199,27 +176,18 @@ private List doPlanScan(ConnectorSession session, ConnectorS boolean isLzo = isLzoInputFormat(hiveHandle.getInputFormat()); // LZO text is NOT splittable: a .lzo stream cannot be decompressed from an arbitrary byte offset. // Legacy HiveUtil.isSplittable returned false for LZO; HiveFileFormat maps LZO text to TEXT (which - // reports splittable), so the LZO case must be masked out here. ACID is never LZO, so this leaves the - // transactional branch unchanged. + // reports splittable), so the LZO case must be masked out here. boolean splittable = fileFormat.isSplittable() && !isLzo; List ranges = new ArrayList<>(); - if (hiveHandle.isTransactional()) { - // Transactional (ACID) table: descend into base/delta directories under the query's write-id - // snapshot and emit ACID-annotated ranges. Borrows the engine's per-catalog Doris FileSystem to - // list (same source as the non-ACID branch below; the engine owns its lifecycle — never closed here). - planAcidScan(session, hiveHandle, partitions, storage().getFileSystem(session), fileFormat, - splittable, targetSplitSize, ranges); - } else { - // Borrow the engine's per-catalog Doris FileSystem to list partition directories (see field javadoc). - FileSystem fs = storage().getFileSystem(session); - for (PartitionScanInfo partition : partitions) { - HiveFileFormat partFormat = partition.fileFormat != null - ? partition.fileFormat : fileFormat; - listAndSplitFiles(dbName, tableName, partition, partFormat, - splittable, isLzo, targetSplitSize, fs, ranges, completeness); - } + // Borrow the engine's per-catalog Doris FileSystem to list partition directories (see field javadoc). + FileSystem fs = storage().getFileSystem(session); + for (PartitionScanInfo partition : partitions) { + HiveFileFormat partFormat = partition.fileFormat != null + ? partition.fileFormat : fileFormat; + listAndSplitFiles(dbName, tableName, partition, partFormat, + splittable, isLzo, targetSplitSize, fs, ranges, completeness); } LOG.info("Hive scan plan: table={}.{}, partitions={}, splits={}", @@ -235,25 +203,21 @@ private List doPlanScan(ConnectorSession session, ConnectorS * partition's files synchronously (legacy {@code HiveScanNode} went async at * {@code num_partitions_in_batch_mode}). * - *

    Transactional/ACID tables are excluded DELIBERATELY: their scan opens one metastore read transaction - * (see {@link #planAcidScan}); running the per-batch resolution on background threads would open — and leak - * — a read transaction per batch. ACID partitioned tables keep the synchronous {@link #planScan} path - * (correct, just not streamed). The transactional test uses the SAME {@code isTransactional()} accessor - * {@link #planScan} branches on.

    */ @Override public boolean supportsBatchScan(ConnectorSession session, ConnectorTableHandle handle) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; + rejectTransactionalRead(hiveHandle); List partKeyNames = hiveHandle.getPartitionKeyNames(); - return partKeyNames != null && !partKeyNames.isEmpty() && !hiveHandle.isTransactional(); + return partKeyNames != null && !partKeyNames.isEmpty(); } @Override public long streamingSplitEstimate(ConnectorSession session, ConnectorTableHandle handle, Optional filter, boolean countPushdown) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; - if (hiveHandle.isTransactional() - || Boolean.FALSE.equals(session.getProperty("enable_external_table_batch_mode", Boolean.class))) { + rejectTransactionalRead(hiveHandle); + if (Boolean.FALSE.equals(session.getProperty("enable_external_table_batch_mode", Boolean.class))) { return -1; } int fileThreshold = sessionInt(session, "num_files_in_batch_mode", DEFAULT_BATCH_THRESHOLD); @@ -291,12 +255,18 @@ private long cachedFileCount(HiveTableHandle handle) { @Override public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorScanRequest request) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); - if (hiveHandle.isTransactional()) { - throw new DorisConnectorException("Transactional Hive scans require snapshot-aware planning"); - } + rejectTransactionalRead(hiveHandle); return new HiveSplitSource(session, hiveHandle, request.getRequiredPartitions()); } + private static void rejectTransactionalRead(HiveTableHandle handle) { + // Both full ACID and insert-only layouts need write-id visibility; plain file scans + // (including metadata-only counts) cannot return a transactionally correct result. + if (handle.isTransactional()) { + throw new DorisConnectorException("Hive transactional table reads are no longer supported"); + } + } + private static int sessionInt(ConnectorSession session, String name, int defaultValue) { Integer value = session.getProperty(name, Integer.class); return value == null ? defaultValue : value; @@ -320,6 +290,7 @@ private final class HiveSplitSource implements ConnectorSplitSource { private volatile boolean closed; HiveSplitSource(ConnectorSession session, HiveTableHandle handle, List requiredPartitions) { + rejectTransactionalRead(handle); this.handle = handle; this.partitionNames = requiredPartitions.isEmpty() ? null : requiredPartitions; this.fs = storage().getFileSystem(session); @@ -361,7 +332,7 @@ private void loadNextFileRanges() { } HiveFileFormat format = partition.fileFormat == null ? tableFormat : partition.fileFormat; ranges = splitFileRanges(file.getPath(), file.getLength(), file.getModificationTime(), - partition, format, splittable, targetSplitSize, null, null); + partition, format, splittable, targetSplitSize); } private void loadNextPartitionFiles() { @@ -452,8 +423,7 @@ public TFileCompressType adjustFileCompressType(TFileCompressType inferred) { * {@link #planScan} per batch) would emit every partition's files once per batch — duplicate rows. The batch * strings are the metastore-rendered {@code key=value/...} partition names (the keys of the Nereids * selected-partition map), exactly the form {@code hmsClient.getPartitions} accepts, so batch-scoped - * resolution is a clean {@code getPartitions(batch)} round-trip. Only the non-ACID path is reachable here - * ({@link #supportsBatchScan} excludes transactional tables). Collects the same partition-scoped source + * resolution is a clean {@code getPartitions(batch)} round-trip. Collects the same partition-scoped source * that {@link #streamSplits} consumes lazily, preserving its file filtering and split boundaries. */ @Override @@ -478,87 +448,6 @@ public List planScanForPartitionBatch( return ranges; } - /** - * Plans a scan of a transactional (ACID) Hive table. - * - *

    Opens (or reuses) the query's read transaction — which acquires a shared read lock and pins a - * write-id snapshot — then, per partition, runs {@link HiveAcidUtil#getAcidState} to resolve the - * surviving base/delta data files and delete-delta directories, and emits one ACID-annotated - * {@link HiveScanRange} per data-file split. The BE subtracts the delete deltas on read.

    - * - *

    Live path. Post-cutover an {@code hms} catalog is plugin-driven, so an hms-catalog - * transactional table is a {@code PluginDrivenExternalTable} routed through {@code PluginDrivenScanNode} - * straight into this method on a live query (the only read gate is the full-ACID ORC-format check below). - * It opens a real metastore transaction/lock; the matching commit (lock release) is driven by - * {@link HiveReadTransactionManager#deregister} at query finish (via {@link #releaseReadTransaction}), - * without which the shared read lock would leak for the metastore's lifetime.

    - */ - private void planAcidScan(ConnectorSession session, HiveTableHandle handle, - List partitions, FileSystem fs, HiveFileFormat fileFormat, - boolean splittable, long targetSplitSize, List ranges) { - boolean isFullAcid = handle.isFullAcid(); - if (isFullAcid && !ORC_ACID_INPUT_FORMAT.equals(handle.getInputFormat())) { - // Full-ACID data is only stored in the ORC ACID layout; reject other formats loudly - // (legacy parity: HMSExternalTable.isFullAcidTable throws "no Orc Format"). - throw new DorisConnectorException("This table is full Acid Table, but no Orc Format."); - } - - // One transaction per transactional-table scan, pinning this table's write-id snapshot (mirrors - // fe-core, which opens a HiveTransaction per scan node). The manager holds it for commit at finish. - HiveReadTransaction txn = new HiveReadTransaction(session.getQueryId(), session.getUser(), - handle.getDbName(), handle.getTableName(), isFullAcid, hmsClient); - readTxnManager.register(txn); - for (PartitionScanInfo partition : partitions) { - if (!partition.partitionValues.isEmpty()) { - txn.addPartition(buildPartitionName(partition.partitionValues)); - } - } - Map txnValidIds = txn.getValidWriteIds(); - - for (PartitionScanInfo partition : partitions) { - HiveFileFormat partFormat = partition.fileFormat != null - ? partition.fileFormat : fileFormat; - try { - // Descend the ACID directory tree through the engine-injected Doris FileSystem. The hive plugin - // bundles no HDFS impl, so a bare Hadoop FileSystem.get here would throw "No FileSystem for - // scheme hdfs" (FIX-HIVEFS); the engine owns this FileSystem's lifecycle — never closed here. - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - fs, partition.location, txnValidIds, isFullAcid); - // Only full-ACID reads are marked transactional_hive (delete deltas applied by the BE - // merge-on-read reader). Insert-only tables use the write-id snapshot to pick files but - // store plain data files, so their splits stay plain hive — matching legacy AcidUtil, - // which sets acidInfo only when isFullAcid. - // BE-facing ACID paths (delete-delta dir + partition location) also carry the raw HMS scheme; the - // BE transactional reader opens them via the same native S3 factory, so normalize s3a://->s3://. - String acidLocation = isFullAcid ? normalizeNativeUri(partition.location) : null; - List encodedDeltas = isFullAcid - ? encodeDeleteDeltas(state.getDeleteDeltas(), this::normalizeNativeUri) : null; - for (FileEntry dataFile : state.getDataFiles()) { - splitFile(dataFile.location().uri(), dataFile.length(), - dataFile.modificationTime(), partition, partFormat, splittable, - targetSplitSize, acidLocation, encodedDeltas, ranges); - } - } catch (IOException e) { - throw new DorisConnectorException( - "Failed to list ACID files for partition: " + partition.location, e); - } - } - } - - /** - * Commits and deregisters this query's read transaction (opened by {@link #planAcidScan} via - * {@link HiveReadTransactionManager#register}), releasing the metastore shared read lock. Driven by the - * generic fe-core query-finish callback at query end; without it a transactional-hive read leaks the shared - * read lock for the metastore's lifetime. {@code readTxnManager} is the same per-connector manager that - * {@code register} used (both injected by {@code HiveConnector}), so the {@code queryId} keys match. - * {@code deregister} is idempotent (a no-op for a query that opened no transaction) and swallows a commit - * failure, matching the best-effort SPI contract. - */ - @Override - public void releaseReadTransaction(String queryId) { - readTxnManager.deregister(queryId); - } - @Override public List collectScanProfiles(ConnectorSession session) { List profiles = new ArrayList<>(partitionBatchProfile.drain()); @@ -579,31 +468,6 @@ private static PartitionBatchProfile statementPruningFailureProfile(ConnectorSes return session.getStatementScope().computeIfAbsent(key, PartitionBatchProfile::new); } - /** Encodes each delete-delta as {@code "dir|file1,file2"} for {@link HiveScanRange.Builder#acidInfo}. */ - private static List encodeDeleteDeltas(List deltas, - UnaryOperator nativePathNormalizer) { - List encoded = new ArrayList<>(deltas.size()); - for (HiveAcidUtil.DeleteDelta delta : deltas) { - // Normalize the delete-delta directory scheme (s3a://->s3://) for BE's native reader; the file names - // are bare names appended after '|'. - encoded.add(nativePathNormalizer.apply(delta.getDirectoryLocation()) + "|" - + String.join(",", delta.getFileNames())); - } - return encoded; - } - - /** Builds the Hive partition name {@code k1=v1/k2=v2} used for the shared read lock components. */ - private static String buildPartitionName(Map partitionValues) { - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : partitionValues.entrySet()) { - if (sb.length() > 0) { - sb.append('/'); - } - sb.append(entry.getKey()).append('=').append(entry.getValue()); - } - return sb.toString(); - } - @Override public Map getScanNodeProperties( ConnectorSession session, @@ -611,6 +475,7 @@ public Map getScanNodeProperties( List columns, Optional filter) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; + rejectTransactionalRead(hiveHandle); Map props = new HashMap<>(); // File format type @@ -658,39 +523,9 @@ public Map getScanNodeProperties( props.putAll(textProps); } - // #65437: BE selects the file scanner from SCAN-LEVEL params before the per-range splits arrive, and its - // FileScannerV2 gate excludes scans whose scan-level table_format_type is "transactional_hive" (V2 does - // not apply ACID delete deltas). Signal a full-ACID scan so populateScanLevelParams stamps it at scan - // level; gate on isFullAcid() to match the per-range marker (insert-only tables keep the V2 fast path). - if (hiveHandle.isFullAcid()) { - props.put(PROP_TRANSACTIONAL_HIVE, "true"); - } - return props; } - /** - * Stamps the SCAN-LEVEL table format for a full-ACID Hive scan (signalled by {@link #PROP_TRANSACTIONAL_HIVE} - * from {@link #getScanNodeProperties}). BE's FileScannerV2 selection reads {@code table_format_type} from the - * scan-level params (before per-range splits are fetched); without this stamp a transactional Hive read would - * wrongly use FileScannerV2, which does not apply ACID delete deltas — deleted rows would reappear. Mirrors the - * legacy {@code HiveScanNode.markTransactionalHiveScanParams} (#65437); the generic {@code PluginDrivenScanNode} - * invokes this hook after building the scan-level params. - */ - @Override - public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { - if ("true".equals(nodeProperties.get(PROP_TRANSACTIONAL_HIVE))) { - markTransactionalHiveScanParams(params); - } - } - - static void markTransactionalHiveScanParams(TFileScanRangeParams params) { - TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); - // Same literal the per-range marker uses (HiveScanRange); BE matches table_format_type == "transactional_hive". - tableFormatFileDesc.setTableFormatType("transactional_hive"); - params.setTableFormatParams(tableFormatFileDesc); - } - /** * Reads the {@code read_hive_json_in_one_column} session flag (default false) from the connector session — * the same channel the write path uses for {@code hive_text_compression}. The engine dumps every visible @@ -792,7 +627,7 @@ private void listAndSplitFiles(String dbName, String tableName, continue; } splitFile(file.getPath(), file.getLength(), file.getModificationTime(), - partition, fileFormat, splittable, targetSplitSize, null, null, ranges); + partition, fileFormat, splittable, targetSplitSize, ranges); } } @@ -891,21 +726,16 @@ private String normalizeNativeUri(String rawUri) { /** * Splits a file into scan ranges based on target split size. - * - *

    When {@code acidPartitionLocation} is non-null the ranges carry ACID delete-delta info - * (marking them {@code transactional_hive}); otherwise they are plain Hive ranges.

    */ private void splitFile(String filePath, long fileSize, long modTime, PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, - String acidPartitionLocation, List acidDeltas, List ranges) { splitFileRanges(filePath, fileSize, modTime, partition, fileFormat, splittable, - targetSplitSize, acidPartitionLocation, acidDeltas).forEachRemaining(ranges::add); + targetSplitSize).forEachRemaining(ranges::add); } private Iterator splitFileRanges(String filePath, long fileSize, long modTime, - PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, - String acidPartitionLocation, List acidDeltas) { + PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize) { if (fileSize == 0) { return Collections.emptyIterator(); } @@ -931,7 +761,7 @@ public ConnectorScanRange next() { } long length = Math.min(splitSize, fileSize - offset); ConnectorScanRange range = newRangeBuilder(normalizedPath, offset, length, fileSize, - modTime, fileFormat, partition, acidPartitionLocation, acidDeltas).build(); + modTime, fileFormat, partition).build(); offset += length; return range; } @@ -939,8 +769,7 @@ public ConnectorScanRange next() { } private static HiveScanRange.Builder newRangeBuilder(String filePath, long start, long length, - long fileSize, long modTime, HiveFileFormat fileFormat, PartitionScanInfo partition, - String acidPartitionLocation, List acidDeltas) { + long fileSize, long modTime, HiveFileFormat fileFormat, PartitionScanInfo partition) { HiveScanRange.Builder builder = HiveScanRange.builder() .path(filePath) .start(start) @@ -950,10 +779,6 @@ private static HiveScanRange.Builder newRangeBuilder(String filePath, long start .fileFormat(fileFormat.getFormatName()) .tableFormatType("hive") .partitionValues(partition.partitionValues); - if (acidPartitionLocation != null) { - // Sets tableFormatType="transactional_hive" and attaches the delete-delta descriptors. - builder.acidInfo(acidPartitionLocation, acidDeltas); - } return builder; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java index 9e761832669436..6995ed4366cc2c 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java @@ -21,13 +21,9 @@ import org.apache.doris.connector.spi.scan.ConnectorScanRange; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TTableFormatFileDesc; -import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; -import org.apache.doris.thrift.TTransactionalHiveDesc; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,9 +35,6 @@ * Unlike JNI-based scan ranges (ES, JDBC, MaxCompute), Hive scan ranges * carry real file paths with byte offsets, native file format types * (PARQUET/ORC/TEXT), and partition values.

    - * - *

    For ACID tables, additional properties carry transaction info - * (partition location, delete delta paths).

    */ public class HiveScanRange implements ConnectorScanRange { @@ -137,11 +130,6 @@ public String toString() { @Override public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { - if ("transactional_hive".equals(getTableFormatType())) { - populateTransactionalHiveParams(formatDesc); - } - // Non-transactional hive needs no per-split TTableFormatFileDesc fields. - // Rewrite columns-from-path from the connector's partition values, mirroring // IcebergScanRange/PaimonScanRange. This connector now OWNS the Hive default-partition // sentinel (__HIVE_DEFAULT_PARTITION__ -> SQL NULL) mapping that fe-core's @@ -177,46 +165,6 @@ public void populateRangeParams(TTableFormatFileDesc formatDesc, } } - private void populateTransactionalHiveParams(TTableFormatFileDesc formatDesc) { - Map props = getProperties(); - TTransactionalHiveDesc txnDesc = new TTransactionalHiveDesc(); - - String partLoc = props.get("acid.partition_location"); - if (partLoc != null) { - txnDesc.setPartition(partLoc); - } - - String countStr = props.get("acid.delete_delta_count"); - if (countStr != null) { - int count = Integer.parseInt(countStr); - List deltas = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - String deltaStr = props.get("acid.delete_delta." + i); - if (deltaStr != null) { - TTransactionalHiveDeleteDeltaDesc delta = - new TTransactionalHiveDeleteDeltaDesc(); - // Encoded as "dir|file1,file2" (see Builder#acidInfo). BE needs BOTH the - // delete-delta directory AND the file names to correctly apply row deletes; - // dropping the file names silently under-deletes on ACID reads. - int sep = deltaStr.indexOf('|'); - if (sep >= 0) { - delta.setDirectoryLocation(deltaStr.substring(0, sep)); - String fileNamesPart = deltaStr.substring(sep + 1); - if (!fileNamesPart.isEmpty()) { - delta.setFileNames(Arrays.asList(fileNamesPart.split(","))); - } - } else { - delta.setDirectoryLocation(deltaStr); - } - deltas.add(delta); - } - } - txnDesc.setDeleteDeltas(deltas); - } - - formatDesc.setTransactionalHiveParams(txnDesc); - } - public static Builder builder() { return new Builder(); } @@ -289,28 +237,6 @@ public Builder properties(Map val) { return this; } - /** - * Adds ACID-related properties for transactional Hive tables. - * - * @param partitionLocation the partition directory path - * @param deleteDeltas list of delete delta descriptors (dir|file1,file2) - */ - public Builder acidInfo(String partitionLocation, List deleteDeltas) { - if (this.properties == null) { - this.properties = new HashMap<>(); - } - this.tableFormatType = "transactional_hive"; - this.properties.put("acid.partition_location", partitionLocation); - if (deleteDeltas != null) { - this.properties.put("acid.delete_delta_count", - String.valueOf(deleteDeltas.size())); - for (int i = 0; i < deleteDeltas.size(); i++) { - this.properties.put("acid.delete_delta." + i, deleteDeltas.get(i)); - } - } - return this; - } - public HiveScanRange build() { return new HiveScanRange(this); } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java deleted file mode 100644 index 7be2319459b6a1..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java +++ /dev/null @@ -1,263 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsAcidConstants; -import org.apache.doris.filesystem.FileEntry; -import org.apache.doris.filesystem.FileSystem; -import org.apache.doris.filesystem.Location; -import org.apache.doris.filesystem.local.LocalFileSystem; - -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.util.BitSet; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Tests the pure ACID directory descent {@link HiveAcidUtil#getAcidState} against a real Doris - * {@link FileSystem} (the test-only {@link LocalFileSystem}) over a local temp tree — the Doris equivalent of the - * old Hadoop {@code LocalFileSystem}, now that {@link HiveAcidUtil} lists via the engine-injected Doris filesystem. - * - *

    WHY: for a transactional Hive table the reader must reconstruct the correct snapshot from the - * base/delta/delete-delta directory layout — pick the best valid base, layer on the deltas whose - * write-id range is still valid, drop obsolete/out-of-snapshot directories, and hand the BE the - * delete-delta file names so it can subtract deleted rows. A wrong base or a dropped delta silently - * returns stale or over-/under-deleted data. These tests pin that selection algorithm.

    - * - *

    The filesystem is a {@link LiteralListingLocalFileSystem}: it forbids the glob-aware - * {@link FileSystem#listFiles}, so every test here also pins that {@link HiveAcidUtil} lists via the literal - * {@link FileSystem#list} (matching the old {@code FileSystem.listStatus}).

    - */ -public class HiveAcidUtilTest { - - @TempDir - java.nio.file.Path tempDir; - - private FileSystem localFs() { - return new LiteralListingLocalFileSystem(); - } - - /** Creates {@code //} with 1 byte of content. */ - private void createBucketFile(String dir, String fileName) throws IOException { - java.nio.file.Path d = tempDir.resolve(dir); - Files.createDirectories(d); - Files.write(d.resolve(fileName), new byte[] {1}); - } - - /** Snapshot with all visibility txns valid and write-ids valid up to {@code highWatermark}. */ - private Map snapshot(long highWatermark) { - ValidTxnList validTxnList = - new ValidReadTxnList(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); - ValidWriteIdList validWriteIdList = - new ValidReaderWriteIdList("db.tbl", new long[0], new BitSet(), highWatermark); - Map ids = new HashMap<>(); - ids.put(HmsAcidConstants.VALID_TXNS_KEY, validTxnList.writeToString()); - ids.put(HmsAcidConstants.VALID_WRITEIDS_KEY, validWriteIdList.writeToString()); - return ids; - } - - @Test - public void testBestBaseWithDeltaAndDeleteDelta() throws IOException { - // Best base = base_5 (base_2 superseded); delta_6 layered on; delete_delta_6 survives via the - // split-update pairing with delta_6; delta_9 is out of the write-id=6 snapshot; the staging dir - // and the _flush_length side file are ignored. - createBucketFile("base_0000005", "bucket_00000"); - createBucketFile("base_0000002", "bucket_00000"); - createBucketFile("delta_0000006_0000006", "bucket_00000"); - createBucketFile("delta_0000006_0000006", "bucket_00000_flush_length"); - createBucketFile("delete_delta_0000006_0000006", "bucket_00000"); - createBucketFile("delta_0000009_0000009", "bucket_00000"); - createBucketFile(".hive-staging_hive_2026-01-01", "junk"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(6L), true); - - List dataFiles = state.getDataFiles(); - Assertions.assertEquals(2, dataFiles.size(), - "surviving data files must be exactly base_5 + delta_6 bucket files"); - boolean hasBase5 = false; - boolean hasDelta6 = false; - for (FileEntry f : dataFiles) { - String p = f.location().uri(); - Assertions.assertFalse(p.contains("base_0000002"), "superseded base must be dropped: " + p); - Assertions.assertFalse(p.contains("delta_0000009"), "out-of-snapshot delta dropped: " + p); - Assertions.assertFalse(p.endsWith("_flush_length"), "side file excluded: " + p); - hasBase5 |= p.contains("/base_0000005/bucket_00000"); - hasDelta6 |= p.contains("/delta_0000006_0000006/bucket_00000"); - } - Assertions.assertTrue(hasBase5, "best base_5 bucket file must survive"); - Assertions.assertTrue(hasDelta6, "delta_6 bucket file must survive"); - - List deletes = state.getDeleteDeltas(); - Assertions.assertEquals(1, deletes.size(), "the paired delete_delta_6 must survive"); - HiveAcidUtil.DeleteDelta d = deletes.get(0); - Assertions.assertTrue(d.getDirectoryLocation().endsWith("/delete_delta_0000006_0000006"), - "delete-delta dir: " + d.getDirectoryLocation()); - Assertions.assertEquals(List.of("bucket_00000"), d.getFileNames(), - "delete-delta file names must be captured (BE under-deletes without them)"); - } - - @Test - public void testInsertOnlyRejectsDeleteDelta() throws IOException { - // An insert-only ACID table must never carry delete deltas; a stray one is a corruption signal. - createBucketFile("base_0000005", "000000_0"); - createBucketFile("delta_0000006_0000006", "000000_0"); - createBucketFile("delete_delta_0000006_0000006", "000000_0"); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(6L), false)); - Assertions.assertTrue(ex.getMessage().contains("delete_delta"), - "insert-only table with a delete delta must fail loud: " + ex.getMessage()); - } - - @Test - public void testInsertOnlyAcceptsAllFileNames() throws IOException { - // Insert-only uses the accept-all filter: files that are not bucket_* still count as data. - createBucketFile("base_0000005", "000000_0"); - createBucketFile("delta_0000006_0000006", "000001_0"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(6L), false); - Assertions.assertEquals(2, state.getDataFiles().size(), - "insert-only accepts non-bucket_ data file names"); - Assertions.assertTrue(state.getDeleteDeltas().isEmpty()); - } - - @Test - public void testMissingValidWriteIdsThrows() throws IOException { - createBucketFile("base_0000005", "bucket_00000"); - Map ids = snapshot(6L); - ids.remove(HmsAcidConstants.VALID_WRITEIDS_KEY); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), ids, true)); - Assertions.assertTrue(ex.getMessage().contains("ValidWriteIdList"), ex.getMessage()); - } - - @Test - public void testOriginalFilesWithoutBaseThrows() throws IOException { - // A bare file directly under the partition (no base_) means an unconverted non-ACID table. - Files.write(tempDir.resolve("000000_0"), new byte[] {1}); - - Assertions.assertThrows(UnsupportedOperationException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(6L), true)); - } - - @Test - public void testBaseWithUncommittedVisibilityTxnIsSkipped() throws IOException { - // base_5 was produced by visibility txn 100, which is NOT committed in this snapshot -> it must - // be skipped, falling back to the older committed base_3. This pins the visibility-txn filter. - createBucketFile("base_0000005_v0000100", "bucket_00000"); - createBucketFile("base_0000003", "bucket_00000"); - - // Txn high-watermark 50: visibility txn 100 is not yet valid; base_3's visibility (0) is valid. - ValidTxnList txns = new ValidReadTxnList(new long[0], new BitSet(), 50L, Long.MAX_VALUE); - ValidWriteIdList writeIds = new ValidReaderWriteIdList("db.tbl", new long[0], new BitSet(), 5L); - Map ids = new HashMap<>(); - ids.put(HmsAcidConstants.VALID_TXNS_KEY, txns.writeToString()); - ids.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIds.writeToString()); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), ids, true); - Assertions.assertEquals(1, state.getDataFiles().size()); - Assertions.assertTrue(state.getDataFiles().get(0).location().uri().contains("/base_0000003/"), - "a base whose visibility txn is uncommitted must be skipped for the committed base"); - } - - @Test - public void testHighestBaseInvalidFallsBackToLowerValidBaseAndDropsObsoleteDelta() throws IOException { - // base_8 is beyond the write-id watermark (invalid) and must be rejected despite its higher - // write id, falling back to base_4; delta_2 predates base_4 so it is obsolete and dropped. This - // pins isValidBase discrimination + the selection loop's "delta below the base" rejection. - createBucketFile("base_0000008", "bucket_00000"); - createBucketFile("base_0000004", "bucket_00000"); - createBucketFile("delta_0000002_0000002", "bucket_00000"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(5L), true); - - Assertions.assertEquals(1, state.getDataFiles().size(), - "only the best VALID base (base_4) survives; the higher but invalid base_8 is rejected"); - String p = state.getDataFiles().get(0).location().uri(); - Assertions.assertTrue(p.contains("/base_0000004/"), p); - Assertions.assertFalse(p.contains("base_0000008"), "base above the write-id watermark is invalid"); - } - - @Test - public void testNoValidBaseThrowsNotEnoughHistory() throws IOException { - // Only base_8 exists but it is beyond the write-id watermark -> no usable base and no original - // files -> the reader cannot reconstruct the snapshot and must fail loud. - createBucketFile("base_0000008", "bucket_00000"); - - IOException ex = Assertions.assertThrows(IOException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(5L), true)); - Assertions.assertTrue(ex.getMessage().contains("Not enough history"), ex.getMessage()); - } - - @Test - public void testMissingValidTxnListThrows() throws IOException { - createBucketFile("base_0000005", "bucket_00000"); - Map ids = snapshot(6L); - ids.remove(HmsAcidConstants.VALID_TXNS_KEY); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), ids, true)); - Assertions.assertTrue(ex.getMessage().contains("ValidTxnList"), ex.getMessage()); - } - - /** - * A Doris {@link LocalFileSystem} that FORBIDS the glob-aware {@link FileSystem#listFiles} / - * {@link FileSystem#listFilesRecursive}. Every test lists through this, so any regression in - * {@link HiveAcidUtil} from the literal {@link FileSystem#list} to {@code listFiles()} fails loud here. - * - *

    The production per-scheme filesystems ({@code DFSFileSystem}, {@code S3CompatibleFileSystem}) override - * {@code listFiles} to treat a location containing {@code [}/{@code *}/{@code ?} as a glob pattern; a real hive - * delta/partition location can contain those, and the old {@code FileSystem.listStatus} never glob-expanded. - * Plain {@link LocalFileSystem#listFiles} is the (literal) interface default, so it alone cannot catch a - * {@code list()->listFiles()} regression — hence this guard. Mirrors {@code FakeFileSystem.listFiles} throwing - * in the non-ACID listing tests. - */ - private static final class LiteralListingLocalFileSystem extends LocalFileSystem { - LiteralListingLocalFileSystem() { - super(Collections.emptyMap()); - } - - @Override - public List listFiles(Location dir) { - throw new AssertionError( - "HiveAcidUtil must list via the literal list(), never the glob-aware listFiles()"); - } - - @Override - public List listFilesRecursive(Location dir) { - throw new AssertionError( - "HiveAcidUtil must list via the literal list(), never the glob-aware listFilesRecursive()"); - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java index 1a654fd327e8f6..d5193e2e68d937 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java @@ -515,7 +515,7 @@ public void scanSkipsOnlyTheFailedPartitionAndPlansTheRest() { lister.error = new HiveDirectoryListingException("dir gone", new IOException("boom")); lister.errorLocation = "loc/dt=1"; HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), + new FakeConnectorContext(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); List ranges = provider.planScan(new FakeSession(), @@ -538,7 +538,7 @@ public void scanFailsLoudOnSystemicFilesystemFailure() { CountingLister lister = new CountingLister(); lister.error = new DorisConnectorException("bad storage config"); HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), + new FakeConnectorContext(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(new FakeSession(), @@ -575,7 +575,7 @@ public void scanProviderServesRepeatedScansFromTheCache() { HiveFileListingCache cache = new HiveFileListingCache(HiveTestProperties.minimal(), lister); // hmsClient is null: with pruned partitions on the handle the scan never calls the metastore. HiveScanPlanProvider provider = new HiveScanPlanProvider( - null, HiveTestProperties.minimal(), new FakeConnectorContext(), new HiveReadTransactionManager(), cache); + null, HiveTestProperties.minimal(), new FakeConnectorContext(), cache); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat") diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java deleted file mode 100644 index 44b2dd58363eb5..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java +++ /dev/null @@ -1,231 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsClient; -import org.apache.doris.connector.hms.HmsDatabaseInfo; -import org.apache.doris.connector.hms.HmsPartitionInfo; -import org.apache.doris.connector.hms.HmsTableInfo; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Tests the plugin read-side transaction lifecycle ({@link HiveReadTransaction} / - * {@link HiveReadTransactionManager}) against a recording-fake {@link HmsClient}. - * - *

    WHY: reading a transactional Hive table must open a metastore transaction, take a shared read lock - * exactly once (holding it for the whole scan pins a consistent write-id snapshot), and release it by - * committing when the query finishes. Acquiring the lock more than once, or forgetting to commit, either - * corrupts snapshot isolation or leaks the lock. These tests pin that once-per-query lock + commit - * contract.

    - */ -public class HiveReadTransactionTest { - - @Test - public void testBeginOpensTxnAndValidWriteIdsAcquiresLockOnce() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 777L; - client.validIds.put("hive.txn.valid.writeids", "db.tbl:8:x"); - - HiveReadTransaction txn = new HiveReadTransaction( - "q1", "alice", "db", "tbl", true, client); - txn.begin(); - Assertions.assertEquals(List.of("openTxn:alice"), client.calls); - - txn.addPartition("dt=2026-01-01"); - Map ids = txn.getValidWriteIds(); - Assertions.assertEquals("db.tbl:8:x", ids.get("hive.txn.valid.writeids")); - // The lock must be scoped to the transaction, the (db, table), and the added partitions. - Assertions.assertEquals(777L, client.lockTxnId); - Assertions.assertEquals("db.tbl", client.getValidWriteIdsTable); - Assertions.assertEquals(777L, client.getValidWriteIdsTxnId); - Assertions.assertEquals(List.of("dt=2026-01-01"), client.lockPartitions); - - // Second call must be memoized: no extra lock / getValidWriteIds round-trips. - Map ids2 = txn.getValidWriteIds(); - Assertions.assertSame(ids, ids2); - Assertions.assertEquals(1, count(client.calls, "acquireSharedLock:q1")); - Assertions.assertEquals(1, count(client.calls, "getValidWriteIds:db.tbl")); - } - - @Test - public void testCommitCommitsTheTxn() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 42L; - HiveReadTransaction txn = new HiveReadTransaction("q1", "bob", "db", "t", false, client); - txn.begin(); - txn.commit(); - Assertions.assertTrue(client.calls.contains("commitTxn:42"), client.calls.toString()); - } - - @Test - public void testManagerRegisterBeginsAndDeregisterCommits() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 5L; - HiveReadTransactionManager mgr = new HiveReadTransactionManager(); - - HiveReadTransaction txn = new HiveReadTransaction("q1", "u", "db", "t", true, client); - mgr.register(txn); - Assertions.assertEquals(1, count(client.calls, "openTxn:u"), "register opens the txn"); - - mgr.deregister("q1"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:5"), "deregister commits the txn"); - - // Idempotent: a second deregister and an unknown query must be no-ops. - mgr.deregister("q1"); - mgr.deregister("unknown"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:5"), "no double commit"); - } - - @Test - public void testScanProviderReleaseReadTransactionCommitsViaSharedManager() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 9L; - // register and the provider's release MUST share the same per-connector manager (HiveConnector injects - // one instance into every provider), so the release finds and commits the txn register opened. - HiveReadTransactionManager mgr = new HiveReadTransactionManager(); - HiveScanPlanProvider provider = new HiveScanPlanProvider(client, HiveTestProperties.minimal(), - new FakeConnectorContext(), mgr, new HiveFileListingCache(HiveTestProperties.minimal())); - - // A transactional-hive scan opened a read txn (as planAcidScan does via mgr.register), taking the shared - // read lock. - mgr.register(new HiveReadTransaction("q9", "u", "db", "t", true, client)); - Assertions.assertEquals(1, count(client.calls, "openTxn:u"), "the scan opened a read txn"); - - // The query-finish callback routes through the provider's releaseReadTransaction, which must commit the - // txn (releasing the shared read lock) exactly once — otherwise the lock leaks for the metastore's life. - provider.releaseReadTransaction("q9"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:9"), - "releaseReadTransaction must commit the txn (release the shared read lock) exactly once"); - - // Idempotent: a second release, or a release for a query that opened no txn, is a safe no-op. - provider.releaseReadTransaction("q9"); - provider.releaseReadTransaction("never-opened"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:9"), "no double commit on repeat release"); - } - - private static int count(List calls, String prefix) { - int n = 0; - for (String c : calls) { - if (c.startsWith(prefix)) { - n++; - } - } - return n; - } - - /** - * Recording {@link HmsClient} fake: stubs the abstract read surface and records the four read-ACID - * primitives the read transaction drives. All other primitives keep their default {@code throw}. - */ - private static final class RecordingHmsClient implements HmsClient { - final List calls = new ArrayList<>(); - long openTxnReturn; - long lockTxnId; - List lockPartitions; - long getValidWriteIdsTxnId; - String getValidWriteIdsTable; - final Map validIds = new HashMap<>(); - - @Override - public long openTxn(String user) { - calls.add("openTxn:" + user); - return openTxnReturn; - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - calls.add("acquireSharedLock:" + queryId + ":" + txnId); - this.lockTxnId = txnId; - this.lockPartitions = new ArrayList<>(partitionNames); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - calls.add("getValidWriteIds:" + fullTableName + ":" + currentTransactionId); - this.getValidWriteIdsTxnId = currentTransactionId; - this.getValidWriteIdsTable = fullTableName; - return validIds; - } - - @Override - public void commitTxn(long txnId) { - calls.add("commitTxn:" + txnId); - } - - // ---- unused abstract read surface ---- - - @Override - public List listDatabases() { - return Collections.emptyList(); - } - - @Override - public HmsDatabaseInfo getDatabase(String dbName) { - throw new UnsupportedOperationException("getDatabase"); - } - - @Override - public List listTables(String dbName) { - return Collections.emptyList(); - } - - @Override - public boolean tableExists(String dbName, String tableName) { - return false; - } - - @Override - public HmsTableInfo getTable(String dbName, String tableName) { - throw new UnsupportedOperationException("getTable"); - } - - @Override - public Map getDefaultColumnValues(String dbName, String tableName) { - return Collections.emptyMap(); - } - - @Override - public List listPartitionNames(String dbName, String tableName, int maxParts) { - return Collections.emptyList(); - } - - @Override - public List getPartitions(String dbName, String tableName, - List partNames) { - return Collections.emptyList(); - } - - @Override - public HmsPartitionInfo getPartition(String dbName, String tableName, List values) { - throw new UnsupportedOperationException("getPartition"); - } - - @Override - public void close() { - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java index e4b45ceaa12401..cefeb2b13e985a 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java @@ -37,7 +37,6 @@ import org.apache.doris.filesystem.FileSystem; import org.apache.doris.foundation.format.FileCompression; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileScanRangeParams; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -64,9 +63,8 @@ * *

    WHY (Rule 9): the two overrides encode two correctness-critical decisions.

    *
      - *
    • {@code supportsBatchScan}: batch iff PARTITIONED and NOT transactional. ACID tables are excluded so the - * per-batch resolution does not open (and leak) a metastore read transaction per batch — they keep the - * synchronous path.
    • + *
    • {@code supportsBatchScan}: partitioned tables can use batch planning; transactional tables + * are rejected because their read visibility is no longer supported.
    • *
    • {@code planScanForPartitionBatch}: hive's {@code planScan} resolves partitions from the handle's pruned * set and IGNORES the passed partition set, so hive MUST override the batch hook to scope resolution to the * batch. If it inherited the SPI default (which re-runs the whole-pruned-set {@code planScan} per batch), @@ -106,7 +104,7 @@ public String normalizeStorageUri(String uri) { } }; HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), context, - new HiveReadTransactionManager(), cache); + cache); HiveTableHandle handle = parquetHandle(Collections.emptyList()); try (ConnectorSplitSource source = provider.streamSplits(new FakeSession(), ConnectorScanRequest.builder(handle, Collections.emptyList()) @@ -361,7 +359,7 @@ public void streamingDecisionUsesCachedCountWithoutListing() { return Collections.nCopies(3, new HiveFileStatus(location + "/data.parquet", 10, 1)); }); HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), cache); + new FakeConnectorContext(), cache); HiveTableHandle handle = parquetHandle(Collections.emptyList()); FakeSession session = new FakeSession(Collections.singletonMap("num_files_in_batch_mode", "2")); Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), false) >= 0); @@ -408,7 +406,7 @@ public void supportsBatchScanIsFalseForUnpartitionedTable() { } @Test - public void supportsBatchScanIsFalseForTransactionalPartitionedTable() { + public void supportsBatchScanRejectsTransactionalPartitionedTable() { HiveScanPlanProvider provider = provider(null, new CountingLister()); Map txnParams = new HashMap<>(); txnParams.put("transactional", "true"); @@ -416,10 +414,8 @@ public void supportsBatchScanIsFalseForTransactionalPartitionedTable() { .partitionKeyNames(PART_KEYS) .tableParameters(txnParams) .build(); - // WHY (Rule 9): ACID tables are excluded DELIBERATELY — running planScanForPartitionBatch per batch on - // background threads would open (and leak) a metastore read transaction per batch. They keep the - // synchronous path even though they are partitioned. - Assertions.assertFalse(provider.supportsBatchScan(new FakeSession(), handle)); + Assertions.assertThrows(DorisConnectorException.class, + () -> provider.supportsBatchScan(new FakeSession(), handle)); } // ==================== planScanForPartitionBatch: scoped to the batch, no duplication ==================== @@ -863,41 +859,6 @@ public void partitionBatchPlanningNeverUsesStatementReuse() { "partition batches must not be planned through the statement reuse memo"); } - @Test - public void transactionalScanNeverUsesStatementReuse() { - // Each transactional scan opens its own read transaction pinning a write-id snapshot; a reused plan would - // skip it. The fake metastore refuses the transaction, so planning stops right after asking for it. - // MUTATION: dropping the transactional bypass in planScan -> the scan installs the statement reuse memo - // before planning -> red. - int[] requestedTransactions = new int[1]; - HmsClient hmsClient = new FakeHmsClient() { - @Override - public long openTxn(String user) { - requestedTransactions[0]++; - throw new UnsupportedOperationException("no read transactions in this test"); - } - }; - HiveScanPlanProvider provider = provider(hmsClient, new CountingLister()); - Map insertOnly = new HashMap<>(); - insertOnly.put("transactional", "true"); - insertOnly.put("transactional_properties", "insert_only"); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .partitionKeyNames(PART_KEYS) - .tableParameters(insertOnly) - .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) - .build(); - TestStatementScope scope = new TestStatementScope(); - ConnectorSession session = new ScopeSession(7L, "same-statement", scope); - - Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(session, scanOf(handle))); - - Assertions.assertEquals(1, requestedTransactions[0], "a transactional scan asks for its read transaction"); - Assertions.assertFalse(scope.contains(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE + ":7:same-statement"), - "a transactional scan must not be planned through the statement reuse memo"); - } - private static HiveTableHandle textHandle(String tableName, String serde, HmsPartitionInfo partition) { return new HiveTableHandle.Builder("db", tableName, HiveTableType.HIVE) .inputFormat(TEXT_INPUT_FORMAT) @@ -928,7 +889,7 @@ public String normalizeStorageUri(String rawUri) { } }; HiveScanPlanProvider provider = new HiveScanPlanProvider(new FakeHmsClient(), - HiveTestProperties.minimal(), normCtx, new HiveReadTransactionManager(), + HiveTestProperties.minimal(), normCtx, new HiveFileListingCache(HiveTestProperties.minimal(), s3aLister)); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat(PARQUET_INPUT_FORMAT) @@ -961,8 +922,7 @@ public Map getBackendStorageProperties() { }; HiveScanPlanProvider provider = new HiveScanPlanProvider(new FakeHmsClient(), HiveTestProperties.with("s3.access_key", "aliasAK"), credCtx, - new HiveReadTransactionManager(), - new HiveFileListingCache(HiveTestProperties.minimal(), new CountingLister())); + new HiveFileListingCache(HiveTestProperties.minimal(), new CountingLister())); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat(PARQUET_INPUT_FORMAT) .serializationLib(PARQUET_SERDE) @@ -1001,74 +961,6 @@ public void getScanNodePropertiesUsesCsvTableParametersForPartitionedAndUnpartit } } - // ============ #65437: scan-level transactional_hive marker (FileScannerV2 exclusion) ============ - // Sibling to supportsBatchScan's ACID exclusion above. BE picks FileScannerV2 from SCAN-LEVEL params before the - // per-range splits arrive, and V2 does not apply ACID delete deltas. A full-ACID hive scan must therefore carry - // the "transactional_hive" marker at scan level (not only per-range) so BE keeps it on FileScanner V1 — - // otherwise deleted rows reappear. - - @Test - public void getScanNodePropertiesEmitsTransactionalHiveForFullAcid() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - Map txnParams = new HashMap<>(); - txnParams.put("transactional", "true"); // full-ACID: transactional AND not insert_only - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .tableParameters(txnParams) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - // WHY (Rule 9): drives the scan-level stamp so BE excludes the full-ACID read from FileScannerV2. - Assertions.assertEquals("true", props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void getScanNodePropertiesOmitsTransactionalHiveForInsertOnly() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - Map params = new HashMap<>(); - params.put("transactional", "true"); - params.put("transactional_properties", "insert_only"); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .tableParameters(params) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - // Insert-only ACID reads have no delete deltas, so FileScannerV2 is correct for them: the marker gates on - // isFullAcid() (NOT isTransactional()), so it must be ABSENT here to keep the V2 fast path. - Assertions.assertNull(props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void getScanNodePropertiesOmitsTransactionalHiveForNonTransactional() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - Assertions.assertNull(props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void populateScanLevelParamsStampsTransactionalHiveOnlyWhenSignalled() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - // With the signal -> scan-level table_format_type = "transactional_hive" (the exact string BE matches on, - // identical to the per-range HiveScanRange marker). - TFileScanRangeParams marked = new TFileScanRangeParams(); - provider.populateScanLevelParams(marked, - Collections.singletonMap(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE, "true")); - Assertions.assertTrue(marked.isSetTableFormatParams()); - Assertions.assertEquals("transactional_hive", marked.getTableFormatParams().getTableFormatType()); - // Without the signal -> untouched, so non-ACID hive keeps the FileScannerV2 fast path. - TFileScanRangeParams unmarked = new TFileScanRangeParams(); - provider.populateScanLevelParams(unmarked, Collections.emptyMap()); - Assertions.assertFalse(unmarked.isSetTableFormatParams()); - } - // ===== helpers ===== @Test @@ -1101,7 +993,7 @@ public void usesHiveParquetInt96TimeZone() { private static HiveScanPlanProvider provider( HmsClient hmsClient, HiveFileListingCache.DirectoryLister lister) { return new HiveScanPlanProvider(hmsClient, HiveTestProperties.minimal(), new FakeConnectorContext(), - new HiveReadTransactionManager(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); + new HiveFileListingCache(HiveTestProperties.minimal(), lister)); } private static HmsPartitionInfo part(String name) { diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java deleted file mode 100644 index 5ba592feb1967e..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.thrift.TFileRangeDesc; -import org.apache.doris.thrift.TTableFormatFileDesc; -import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; -import org.apache.doris.thrift.TTransactionalHiveDesc; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.List; - -/** - * Tests that {@link HiveScanRange} carries ACID delete-delta info to the BE. - * - *

      WHY: for transactional Hive tables the BE applies row-level deletes by reading the - * delete-delta files listed per partition. The delta descriptor needs BOTH the directory - * AND the file names within it. If the file names are dropped, the BE cannot locate the - * delete records and silently under-deletes — returning rows that were logically deleted. - * These tests pin the "dir|file1,file2" encode/decode round-trip end to end.

      - */ -public class HiveScanRangeAcidTest { - - @Test - public void testDeleteDeltaCarriesDirectoryAndFileNames() { - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/delta_0000005_0000005/bucket_00000") - .acidInfo("/tbl/p=1", Arrays.asList( - "/tbl/p=1/delete_delta_0000003_0000003|bucket_00000,bucket_00001", - "/tbl/p=1/delete_delta_0000004_0000004|bucket_00000")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - Assertions.assertTrue(formatDesc.isSetTransactionalHiveParams(), - "transactional_hive range must emit transactional params"); - TTransactionalHiveDesc txnDesc = formatDesc.getTransactionalHiveParams(); - Assertions.assertEquals("/tbl/p=1", txnDesc.getPartition()); - - List deltas = txnDesc.getDeleteDeltas(); - Assertions.assertEquals(2, deltas.size()); - - TTransactionalHiveDeleteDeltaDesc first = deltas.get(0); - Assertions.assertEquals("/tbl/p=1/delete_delta_0000003_0000003", - first.getDirectoryLocation()); - // The regression: file names must survive the "dir|file1,file2" round-trip. - Assertions.assertEquals(Arrays.asList("bucket_00000", "bucket_00001"), - first.getFileNames()); - - TTransactionalHiveDeleteDeltaDesc second = deltas.get(1); - Assertions.assertEquals("/tbl/p=1/delete_delta_0000004_0000004", - second.getDirectoryLocation()); - Assertions.assertEquals(Arrays.asList("bucket_00000"), second.getFileNames()); - } - - @Test - public void testDeleteDeltaWithoutFileNamesLeavesFileNamesUnset() { - // A directory-only encoding (no '|') must still set the directory and simply - // carry no file names, rather than mis-parsing the whole string as a directory - // with a bogus trailing file name. - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/base_0000005/bucket_00000") - .acidInfo("/tbl/p=1", Arrays.asList("/tbl/p=1/delete_delta_dir_only")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - TTransactionalHiveDeleteDeltaDesc delta = - formatDesc.getTransactionalHiveParams().getDeleteDeltas().get(0); - Assertions.assertEquals("/tbl/p=1/delete_delta_dir_only", delta.getDirectoryLocation()); - Assertions.assertFalse(delta.isSetFileNames()); - } - - @Test - public void testNonTransactionalRangeEmitsNoTransactionalParams() { - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/000000_0") - .fileFormat("parquet") - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - Assertions.assertFalse(formatDesc.isSetTransactionalHiveParams()); - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java index 490b4daf980b11..13cfd67ac1d401 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java @@ -122,25 +122,4 @@ public void testUnpartitionedEmitsNoColumnsFromPath() { Assertions.assertFalse(rangeDesc.isSetColumnsFromPathIsNull()); } - @Test - public void testAcidTableEmitsBothTransactionalParamsAndPartitionValues() { - // ACID (transactional_hive) ranges must still carry columns-from-path in addition to the - // transactional delete-delta params; the partition-value rebuild runs regardless of ACID. - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/dt=2024-01-01/delta_0000005_0000005/bucket_00000") - .partitionValues(orderedPartitionValues("dt", "2024-01-01")) - .acidInfo("/tbl/dt=2024-01-01", Arrays.asList( - "/tbl/dt=2024-01-01/delete_delta_0000003_0000003|bucket_00000")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - TFileRangeDesc rangeDesc = new TFileRangeDesc(); - range.populateRangeParams(formatDesc, rangeDesc); - - Assertions.assertTrue(formatDesc.isSetTransactionalHiveParams(), - "ACID range must still emit transactional params"); - Assertions.assertEquals(Collections.singletonList("dt"), rangeDesc.getColumnsFromPathKeys()); - Assertions.assertEquals(Collections.singletonList("2024-01-01"), rangeDesc.getColumnsFromPath()); - Assertions.assertEquals(Collections.singletonList(false), rangeDesc.getColumnsFromPathIsNull()); - } } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java new file mode 100644 index 00000000000000..b98c9dac162f7f --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +public class HiveUnsupportedReadsTest { + @Test + public void rejectTransactionalTablesBeforePlanningOrCountPushdown() { + HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), + null, null); + for (String mode : new String[] {"default", "insert_only"}) { + HiveTableHandle handle = new HiveTableHandle.Builder("db", "tbl", HiveTableType.HIVE) + .inputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat") + .tableParameters(Map.of("transactional", "true", "transactional_properties", mode)) + .build(); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.getScanNodeProperties(new ScopeSession(1L, "q", + new TestStatementScope()), handle, Collections.emptyList(), Optional.empty())); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); + Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(null, request)); + Assertions.assertThrows(DorisConnectorException.class, () -> provider.streamSplits(null, request)); + Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planScanForPartitionBatch(null, request, Collections.singletonList("p=1"))); + Assertions.assertTrue(error.getMessage().contains( + "Hive transactional table reads are no longer supported")); + } + } +} diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index f9125dfe66b190..37b5aae4727859 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -604,27 +604,6 @@ public boolean partitionExists(String dbName, String tableName, List par return delegate.partitionExists(dbName, tableName, partitionValues); } - @Override - public long openTxn(String user) { - return delegate.openTxn(user); - } - - @Override - public void commitTxn(long txnId) { - delegate.commitTxn(txnId); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - return delegate.getValidWriteIds(fullTableName, currentTransactionId); - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - delegate.acquireSharedLock(queryId, txnId, user, dbName, tableName, partitionNames, timeoutMs); - } - @Override public long acquireExclusiveTableLock(String queryId, String user, String dbName, String tableName, long timeoutMs) { diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java deleted file mode 100644 index eac80de20042a3..00000000000000 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hms; - -/** - * Hadoop-configuration keys that carry the ACID snapshot (valid transaction list + valid write-id - * list) from FE to BE for transactional Hive reads. - * - *

      These are a fixed Hive/BE contract: the same literal strings fe-core's {@code AcidUtil} - * produces (via {@link HmsClient#getValidWriteIds}) and consumes. Defined once here so the write-id - * producer ({@link ThriftHmsClient}) and the future read-side ACID descent share one source of - * truth.

      - */ -public final class HmsAcidConstants { - - /** Serialized {@code ValidTxnList}. */ - public static final String VALID_TXNS_KEY = "hive.txn.valid.txns"; - - /** Serialized {@code ValidWriteIdList}. */ - public static final String VALID_WRITEIDS_KEY = "hive.txn.valid.writeids"; - - private HmsAcidConstants() { - } -} diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java index ff81a8cfee9be2..1ce842d1a0cbb9 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java @@ -369,66 +369,6 @@ default boolean partitionExists(String dbName, String tableName, throw new UnsupportedOperationException("partitionExists is not supported by this client"); } - // ========== Phase 4: ACID transactions ========== - // - // Read-side ACID primitives for transactional Hive tables. Like the Phase 3 block, they default - // to throwing so read-only / non-transactional clients need not implement them. - - /** - * Open a Hive ACID transaction. - * - * @param user the user opening the transaction - * @return the new transaction id - * @throws HmsClientException if the operation fails - */ - default long openTxn(String user) { - throw new UnsupportedOperationException("openTxn is not supported by this client"); - } - - /** - * Commit a Hive ACID transaction (also releases the transaction's locks). - * - * @param txnId the transaction id - * @throws HmsClientException if the operation fails - */ - default void commitTxn(long txnId) { - throw new UnsupportedOperationException("commitTxn is not supported by this client"); - } - - /** - * Get the valid-transaction / valid-write-id snapshot for a transactional table, as the two - * Hadoop-configuration entries ({@link HmsAcidConstants}) that BE consumes for ACID reads. On a - * metastore-incompatibility error the implementation degrades to a max watermark rather than - * failing the read. - * - * @param fullTableName fully-qualified "db.table" - * @param currentTransactionId the reader's open transaction id - * @return map with {@link HmsAcidConstants#VALID_TXNS_KEY} and - * {@link HmsAcidConstants#VALID_WRITEIDS_KEY} - * @throws HmsClientException if the operation fails - */ - default Map getValidWriteIds(String fullTableName, long currentTransactionId) { - throw new UnsupportedOperationException("getValidWriteIds is not supported by this client"); - } - - /** - * Acquire a shared (read) lock over a table and, if given, specific partitions, polling until the - * lock is granted or the timeout elapses. - * - * @param queryId the query id (lock owner) - * @param txnId the transaction id - * @param user the requesting user - * @param dbName database name - * @param tableName table name - * @param partitionNames partitions to lock; empty locks the whole table - * @param timeoutMs maximum time to wait for the lock, in milliseconds - * @throws HmsClientException if the lock cannot be acquired within the timeout - */ - default void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - throw new UnsupportedOperationException("acquireSharedLock is not supported by this client"); - } - /** * Acquire a standalone exclusive lock over a non-transactional Hive table. * diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java index 9efde69b5a12d1..d9b388d98df0e2 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java @@ -25,11 +25,6 @@ import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -46,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; @@ -60,15 +54,12 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import shade.doris.hive.org.apache.thrift.TApplicationException; import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -532,85 +523,6 @@ public boolean partitionExists(String dbName, String tableName, }); } - // ========== Phase 4: ACID transactions ========== - - @Override - public long openTxn(String user) { - return execute(client -> client.openTxn(user)); - } - - @Override - public void commitTxn(long txnId) { - execute(client -> { - client.commitTxn(txnId); - return null; - }); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - Map conf = new HashMap<>(); - try { - return execute(client -> { - // Use the recent snapshot of valid transactions (no currentTxn arg): passing - // currentTransactionId here would break Hive's delta-directory listing if a major - // compaction removed deltas for transactions that were valid when this txn opened. - ValidTxnList validTransactions = client.getValidTxns(); - List tableValidWriteIdsList = client.getValidWriteIds( - Collections.singletonList(fullTableName), validTransactions.toString()); - if (tableValidWriteIdsList.size() != 1) { - throw new HmsClientException("tableValidWriteIdsList's size should be 1"); - } - ValidTxnWriteIdList validTxnWriteIdList = TxnUtils.createValidTxnWriteIdList( - currentTransactionId, tableValidWriteIdsList); - ValidWriteIdList writeIdList = - validTxnWriteIdList.getTableValidWriteIdList(fullTableName); - conf.put(HmsAcidConstants.VALID_TXNS_KEY, validTransactions.writeToString()); - conf.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIdList.writeToString()); - return conf; - }); - } catch (Exception e) { - // Older / incompatible metastores may lack these APIs; degrade to a max watermark - // instead of failing the scan (mirrors legacy ThriftHMSCachedClient). - LOG.warn("failed to get valid write ids for {}, transaction {}", - fullTableName, currentTransactionId, e); - ValidTxnList validTransactions = new ValidReadTxnList( - new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); - ValidWriteIdList writeIdList = new ValidReaderWriteIdList( - fullTableName, new long[0], new BitSet(), Long.MAX_VALUE); - conf.put(HmsAcidConstants.VALID_TXNS_KEY, validTransactions.writeToString()); - conf.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIdList.writeToString()); - return conf; - } - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - LockRequestBuilder requestBuilder = new LockRequestBuilder(queryId) - .setTransactionId(txnId) - .setUser(user); - for (LockComponent component - : createLockComponentsForRead(dbName, tableName, partitionNames)) { - requestBuilder.addLockComponent(component); - } - LockRequest lockRequest = requestBuilder.build(); - LockResponse response = execute(client -> client.lock(lockRequest)); - long start = System.currentTimeMillis(); - while (response.getState() == LockState.WAITING) { - if (System.currentTimeMillis() - start > timeoutMs) { - throw new HmsClientException("acquire lock timeout for txn " + txnId - + " of query " + queryId + ", timeout(ms): " + timeoutMs); - } - long lockId = response.getLockid(); - response = execute(client -> client.checkLock(lockId)); - } - if (response.getState() != LockState.ACQUIRED) { - throw new HmsClientException( - "failed to acquire lock, lock in state " + response.getState()); - } - } - @Override public long acquireExclusiveTableLock(String queryId, String user, String dbName, String tableName, long timeoutMs) { @@ -700,34 +612,6 @@ public List getNextNotification(long lastEventId, int maxE }); } - private static List createLockComponentsForRead(String dbName, String tableName, - List partitionNames) { - List components = new ArrayList<>( - partitionNames.isEmpty() ? 1 : partitionNames.size()); - if (partitionNames.isEmpty()) { - components.add(createLockComponentForRead(dbName, tableName, null)); - } else { - for (String partitionName : partitionNames) { - components.add(createLockComponentForRead(dbName, tableName, partitionName)); - } - } - return components; - } - - private static LockComponent createLockComponentForRead(String dbName, String tableName, - String partitionName) { - LockComponentBuilder builder = new LockComponentBuilder(); - builder.setShared(); - builder.setOperationType(DataOperationType.SELECT); - builder.setDbName(dbName); - builder.setTableName(tableName); - if (partitionName != null) { - builder.setPartitionName(partitionName); - } - builder.setIsTransactional(true); - return builder.build(); - } - @Override public synchronized void close() throws IOException { if (closed) { diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java index 6ca79cb1beabd0..a3e6cfdbe308f1 100644 --- a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java @@ -18,7 +18,6 @@ package org.apache.doris.connector.hms; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.LockComponent; @@ -29,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import shade.doris.hive.org.apache.thrift.TApplicationException; @@ -38,10 +36,8 @@ import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -65,29 +61,6 @@ */ public class ThriftHmsClientWriteAcidTest { - // ---- openTxn / commitTxn -------------------------------------------------------------------- - - @Test - public void testOpenTxnForwardsUserAndReturnsId() { - RecordingClient fake = new RecordingClient().stub("openTxn", 42L); - ThriftHmsClient client = newClient(fake); - - long txnId = client.openTxn("alice"); - - Assertions.assertEquals(42L, txnId); - Assertions.assertEquals("alice", argsOf(fake, "openTxn")[0]); - } - - @Test - public void testCommitTxnForwardsId() { - RecordingClient fake = new RecordingClient(); - ThriftHmsClient client = newClient(fake); - - client.commitTxn(7L); - - Assertions.assertEquals(7L, argsOf(fake, "commitTxn")[0]); - } - // ---- dropPartition / partitionExists -------------------------------------------------------- @Test @@ -422,116 +395,6 @@ public void testUpdatePartitionStatisticsRejectsAmbiguousName() { current -> HmsPartitionStatistics.EMPTY)); } - // ---- getValidWriteIds ----------------------------------------------------------------------- - - @Test - public void testGetValidWriteIdsSuccessPathEmitsSnapshotThenWriteIds() { - // Primary read-ACID path: a compatible metastore returns exactly one write-id list. A - // distinctive snapshot (high-watermark 100, NOT the fallback's Long.MAX_VALUE) makes this test - // fail if the code silently degrades to the fallback branch. - ValidReadTxnList snapshot = new ValidReadTxnList(new long[0], new BitSet(), 100L, 5L); - TableValidWriteIds writeIds = new TableValidWriteIds( - "db.t", 100L, Collections.emptyList(), ByteBuffer.allocate(0)); - RecordingClient fake = new RecordingClient() - .stub("getValidTxns", snapshot) - .stub("getValidWriteIds", Collections.singletonList(writeIds)); - ThriftHmsClient client = newClient(fake); - - Map conf = client.getValidWriteIds("db.t", 42L); - - // (a) The recent snapshot string — NOT currentTransactionId (42) — drives the write-id lookup. - Object[] gvwiArgs = argsOf(fake, "getValidWriteIds"); - Assertions.assertEquals(Collections.singletonList("db.t"), gvwiArgs[0]); - Assertions.assertEquals(snapshot.toString(), gvwiArgs[1]); - // (b) VALID_TXNS_KEY carries the txn snapshot; VALID_WRITEIDS_KEY carries the write-id list — - // no key swap. Both are the SUCCESS values (snapshot watermark 100), not the fallback watermark. - Assertions.assertEquals(snapshot.writeToString(), conf.get(HmsAcidConstants.VALID_TXNS_KEY)); - Assertions.assertTrue(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY).contains("db.t")); - Assertions.assertNotEquals(conf.get(HmsAcidConstants.VALID_TXNS_KEY), - conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - } - - @Test - public void testGetValidWriteIdsFallsBackToMaxWatermark() { - // An incompatible metastore returns an unexpected write-id list; rather than fail the scan, - // getValidWriteIds must degrade to a max watermark and still emit both config keys. - ValidReadTxnList snapshot = new ValidReadTxnList(); - RecordingClient fake = new RecordingClient() - .stub("getValidTxns", snapshot) - .stub("getValidWriteIds", new ArrayList<>()); // size 0 -> triggers fallback - ThriftHmsClient client = newClient(fake); - - Map conf = client.getValidWriteIds("db.t", 5L); - - // The recent snapshot (not currentTransactionId) was still forwarded before the size guard threw. - Object[] gvwiArgs = argsOf(fake, "getValidWriteIds"); - Assertions.assertEquals(Collections.singletonList("db.t"), gvwiArgs[0]); - Assertions.assertEquals(snapshot.toString(), gvwiArgs[1]); - // Both BE-contract keys are present, and the fallback write-id list is scoped to the table. - Assertions.assertNotNull(conf.get(HmsAcidConstants.VALID_TXNS_KEY)); - Assertions.assertNotNull(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - Assertions.assertTrue(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY).contains("db.t")); - } - - // ---- acquireSharedLock ---------------------------------------------------------------------- - - @Test - public void testAcquireSharedLockBuildsSharedReadComponentsAndReturnsWhenAcquired() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(1L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q1", 5L, "alice", "db", "t", - Collections.emptyList(), 1000L); - - LockRequest request = (LockRequest) argsOf(fake, "lock")[0]; - Assertions.assertEquals(1, request.getComponent().size()); - LockComponent component = request.getComponent().get(0); - Assertions.assertEquals("db", component.getDbname()); - Assertions.assertEquals("t", component.getTablename()); - Assertions.assertEquals(LockType.SHARED_READ, component.getType()); - Assertions.assertEquals(DataOperationType.SELECT, component.getOperationType()); - } - - @Test - public void testAcquireSharedLockOneComponentPerPartition() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(1L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q1", 5L, "alice", "db", "t", - java.util.Arrays.asList("dt=1", "dt=2"), 1000L); - - LockRequest request = (LockRequest) argsOf(fake, "lock")[0]; - Assertions.assertEquals(2, request.getComponent().size()); - Assertions.assertEquals("dt=1", request.getComponent().get(0).getPartitionname()); - Assertions.assertEquals("dt=2", request.getComponent().get(1).getPartitionname()); - } - - @Test - public void testAcquireSharedLockPollsUntilAcquired() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(2L, LockState.WAITING)) - .stub("checkLock", new LockResponse(2L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q", 5L, "u", "db", "t", Collections.emptyList(), 5000L); - - Assertions.assertTrue(fake.methodNames.contains("checkLock")); - } - - @Test - public void testAcquireSharedLockTimesOut() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(3L, LockState.WAITING)) - .stub("checkLock", new LockResponse(3L, LockState.WAITING)); - ThriftHmsClient client = newClient(fake); - - // timeoutMs = -1 => the elapsed guard trips on the first poll iteration, deterministically. - Assertions.assertThrows(HmsClientException.class, () -> - client.acquireSharedLock("q", 5L, "u", "db", "t", Collections.emptyList(), -1L)); - } - @Test public void testExclusiveTableLockIsStandaloneAndExplicitlyReleased() { RecordingClient fake = new RecordingClient() diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java index 42c947c5506776..022ce8c8002e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java @@ -27,7 +27,6 @@ public class FileFormatConstants { public static final String FORMAT_PARQUET = "parquet"; public static final String FORMAT_ORC = "orc"; public static final String FORMAT_JSON = "json"; - public static final String FORMAT_AVRO = "avro"; public static final String FORMAT_WAL = "wal"; public static final String FORMAT_ARROW = "arrow"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java deleted file mode 100644 index 7622cd19644dfb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.fileformat; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.thrift.TFileAttributes; -import org.apache.doris.thrift.TFileFormatType; -import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.doris.thrift.TResultFileSinkOptions; - -import java.util.Map; - -public class AvroFileFormatProperties extends FileFormatProperties { - public AvroFileFormatProperties() { - super(TFileFormatType.FORMAT_AVRO, FileFormatProperties.FORMAT_AVRO); - } - - @Override - public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) - throws AnalysisException { - } - - @Override - public void fullTResultFileSinkOptions(TResultFileSinkOptions sinkOptions) { - } - - @Override - public TFileAttributes toTFileAttributes() { - TFileAttributes fileAttributes = new TFileAttributes(); - TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); - fileAttributes.setTextParams(fileTextScanRangeParams); - return fileAttributes; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index c6b2a8e4109691..cb3cbe0e0bde7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -96,7 +96,7 @@ public static FileFormatProperties createFileFormatProperties(String formatStrin case FORMAT_JSON: return new JsonFileFormatProperties(); case FORMAT_AVRO: - return new AvroFileFormatProperties(); + throw new AnalysisException("Avro file reads are no longer supported"); case FORMAT_WAL: return new WalFileFormatProperties(); case FORMAT_ARROW: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java index 1608a896b99968..88304a130baeb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java @@ -266,7 +266,7 @@ public void createScanRangeLocationsUnsplittable(FileLoadScanNode.ParamCreateCon context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus, fileStatus.size, columnsFromPath.getValues(), columnsFromPathKeys, columnsFromPath.getIsNull()); @@ -313,7 +313,7 @@ public void createScanRangeLocationsSplittable(FileLoadScanNode.ParamCreateConte context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); // Assign scan range locations only for broker load. // stream load has only one file, and no need to set multi scan ranges. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java index 0b6e73ec4f6b34..8b523b240420d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java @@ -66,41 +66,17 @@ public List getIsNull() { */ public static List parseColumnsFromPath(String filePath, List columnsFromPath) throws UserException { - return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, true, false).getValues(); - } - - /** - * Parses partition column values from a Hive-style file path. - * - * @param filePath absolute file path that contains partition segments - * @param columnsFromPath ordered list of partition column names to extract - * @param caseSensitive whether column name matching is case-sensitive - * @param isACID whether the path follows the ACID layout - * ({@code table/par=val/delta_xxx/file}), which adds one extra path level - * @return ordered list of partition values (parallel to {@code columnsFromPath}) - * @throws UserException if the path does not contain the expected partition segments - */ - public static List parseColumnsFromPath( - String filePath, - List columnsFromPath, - boolean caseSensitive, - boolean isACID) - throws UserException { - return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, caseSensitive, isACID) - .getValues(); + return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, true).getValues(); } public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( String filePath, List columnsFromPath, - boolean caseSensitive, - boolean isACID) + boolean caseSensitive) throws UserException { if (columnsFromPath == null || columnsFromPath.isEmpty()) { return new ParsedColumnsFromPath(Collections.emptyList(), Collections.emptyList()); } - // ACID paths have one extra level: table/par=val/delta_xxx/file → pathCount = 3 - int pathCount = isACID ? 3 : 2; List expectedColumns = columnsFromPath; if (!caseSensitive) { expectedColumns = new ArrayList<>(columnsFromPath.size()); @@ -117,13 +93,13 @@ public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( Boolean[] columnValueIsNull = new Boolean[expectedColumns.size()]; int size = 0; boolean skipOnce = true; - for (int i = strings.length - pathCount; i >= 0; i--) { + for (int i = strings.length - 2; i >= 0; i--) { String str = strings[i]; if (str != null && str.isEmpty()) { continue; } if (str == null || !str.contains("=")) { - if (!isACID && skipOnce) { + if (skipOnce) { skipOnce = false; continue; } @@ -164,11 +140,10 @@ public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfoForLoad( String filePath, List columnsFromPath, - boolean caseSensitive, - boolean isACID) + boolean caseSensitive) throws UserException { ParsedColumnsFromPath parsed = parseColumnsFromPathWithNullInfo( - filePath, columnsFromPath, caseSensitive, isACID); + filePath, columnsFromPath, caseSensitive); List values = new ArrayList<>(parsed.getValues()); List isNull = new ArrayList<>(parsed.getIsNull()); for (int i = 0; i < values.size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java index b1d0d4f480e832..921b88077f3929 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java @@ -505,14 +505,11 @@ private TScanRangeLocations splitToScanRange( FileSplit fileSplit = (FileSplit) split; TScanRangeLocations curLocations = newLocations(); // If fileSplit has partition values, use the values collected from hive partitions. - // Otherwise, use the values in file path. Migrated tables emit PluginDrivenSplit (never a HiveSplit) - // and always carry connector-supplied partition values, so isACID stays false: the connector owns - // ACID delta-dir handling and rewrites columns-from-path in its ConnectorScanRange.populateRangeParams. - boolean isACID = false; + // Otherwise, parse partition values from the file path. FilePartitionUtils.ParsedColumnsFromPath partitionValuesFromPath = fileSplit.getPartitionValues() == null ? FilePartitionUtils.parseColumnsFromPathWithNullInfo( - fileSplit.getPathString(), pathPartitionKeys, false, isACID) + fileSplit.getPathString(), pathPartitionKeys, false) : FilePartitionUtils.normalizeColumnsFromPath(fileSplit.getPartitionValues()); TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValuesFromPath.getValues(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java index 841965f63aac6a..45bc68f13b4ba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java @@ -277,7 +277,7 @@ public void createScanRangeLocationsUnsplittable(NereidsParamCreateContext conte context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus, fileStatus.size, columnsFromPath.getValues(), columnsFromPathKeys, columnsFromPath.getIsNull()); @@ -330,7 +330,7 @@ public void createScanRangeLocationsSplittable(NereidsParamCreateContext context context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); // Assign scan range locations only for broker load. // stream load has only one file, and no need to set multi scan ranges. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 637fdf5793c226..ef66650ab5ed23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -839,8 +839,6 @@ public String toString() { public static final String EXCHANGE_MULTI_BLOCKS_BYTE_SIZE = "exchange_multi_blocks_byte_size"; - public static final String SKIP_CHECKING_ACID_VERSION_FILE = "skip_checking_acid_version_file"; - public static final String ENABLE_EXTENDED_REGEX = "enable_extended_regex"; public static final String ENABLE_HYPERSCAN_FALLBACK = "enable_hyperscan_fallback"; @@ -3519,11 +3517,6 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { + "non-utf8 characters will be displayed as garbled characters.") public boolean enableTextValidateUtf8 = true; - @VarAttrDef.VarAttr(name = SKIP_CHECKING_ACID_VERSION_FILE, needForward = true, affectQueryResultInPlan = true, - description = "Skip checking transactional hive version file '_orc_acid_version.'" - ) - public boolean skipCheckingAcidVersionFile = false; - @VarAttrDef.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward = true, checker = "checkSqlConvertorFeatures", description = "enable SQL convertor features. Multiple features are separated by commas") diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java index c473e174189975..7053d256e74296 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java @@ -41,9 +41,8 @@ public HttpStreamTableValuedFunction(Map properties) throws Anal super.parseCommonProperties(properties); if (fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_PARQUET - || fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_AVRO || fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_ORC) { - throw new AnalysisException("http_stream does not yet support parquet, avro and orc"); + throw new AnalysisException("http_stream does not yet support parquet and orc"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java index bbe5e4340ed665..8ad8df14d68e95 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java @@ -133,35 +133,29 @@ public void parseColumnsFromPathPreservesNullMetadataAndInputKeys() throws Excep List partitionKeys = Lists.newArrayList("Region", "Dt"); FilePartitionUtils.ParsedColumnsFromPath parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( "hdfs://host/table/REGION=cn/dt=" + ConnectorPartitionValues.NULL_PARTITION_NAME + "/data.parquet", - partitionKeys, false, false); + partitionKeys, false); Assertions.assertEquals(Lists.newArrayList("cn", ""), parsed.getValues()); Assertions.assertEquals(Lists.newArrayList(false, true), parsed.getIsNull()); Assertions.assertEquals(Lists.newArrayList("Region", "Dt"), partitionKeys); // A literal \N is data on the query path. parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( - "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true, false); + "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true); Assertions.assertEquals(Collections.singletonList("\\N"), parsed.getValues()); Assertions.assertEquals(Collections.singletonList(false), parsed.getIsNull()); - - // An ACID layout has one more directory level between the partition and the file. - parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( - "hdfs://host/table/p=value/delta_1_1/bucket_00000", Collections.singletonList("p"), true, true); - Assertions.assertEquals(Collections.singletonList("value"), parsed.getValues()); - Assertions.assertEquals(Collections.singletonList(false), parsed.getIsNull()); } @Test public void parseColumnsFromPathForLoadKeepsLegacyNullSemantics() throws Exception { FilePartitionUtils.ParsedColumnsFromPath parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad( - "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true, false); + "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true); Assertions.assertEquals(Collections.singletonList(""), parsed.getValues()); Assertions.assertEquals(Collections.singletonList(true), parsed.getIsNull()); // Other values, including the Hive default partition, parse as on the query path. parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad( "hdfs://host/table/p=a/q=" + ConnectorPartitionValues.NULL_PARTITION_NAME + "/data.orc", - Lists.newArrayList("p", "q"), true, false); + Lists.newArrayList("p", "q"), true); Assertions.assertEquals(Lists.newArrayList("a", ""), parsed.getValues()); Assertions.assertEquals(Lists.newArrayList(false, true), parsed.getIsNull()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java new file mode 100644 index 00000000000000..61b7b9947247b9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; + +public class UnsupportedExternalReadsTest { + @Test + public void testAvroFileFormatIsRejected() { + for (String format : new String[] {"avro", "AVRO"}) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> FileFormatProperties.createFileFormatProperties(format)); + Assertions.assertTrue(error.getMessage().contains("Avro file reads are no longer supported")); + } + } + + @Test + public void testLakeSoulCatalogIsRejected() { + CatalogLog log = new CatalogLog(); + log.setCatalogId(1L); + log.setCatalogName("removed_catalog"); + log.setProps(new HashMap<>(Collections.singletonMap("type", "lakesoul"))); + DdlException error = Assertions.assertThrows(DdlException.class, () -> CatalogFactory.createFromLog(log)); + Assertions.assertTrue(error.getMessage().contains("Lakesoul catalog is no longer supported")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java deleted file mode 100644 index 03323d8e159432..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.fileformat; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - - -public class AvroFileFormatPropertiesTest { - - private AvroFileFormatProperties avroFileFormatProperties; - - @BeforeEach - public void setUp() { - avroFileFormatProperties = new AvroFileFormatProperties(); - } - - @Test - public void testAnalyzeFileFormatProperties() { - Map properties = new HashMap<>(); - // Add properties if needed - avroFileFormatProperties.analyzeFileFormatProperties(properties, true); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java index 0be56ce2953520..1bc0e9a004b605 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java @@ -28,7 +28,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.connector.hive.HiveCatalogProperties; import org.apache.doris.connector.hive.HiveFileListingCache; -import org.apache.doris.connector.hive.HiveReadTransactionManager; import org.apache.doris.connector.hive.HiveScanPlanProvider; import org.apache.doris.connector.hive.HiveScanRange; import org.apache.doris.connector.hive.HiveTableHandle; @@ -227,7 +226,7 @@ public void close() { HiveCatalogProperties properties = HiveCatalogProperties.of(Collections.singletonMap( HiveCatalogProperties.HIVE_METASTORE_URIS, "thrift://localhost:9083")); HiveScanPlanProvider provider = new HiveScanPlanProvider(hms, properties, context, - new HiveReadTransactionManager(), new HiveFileListingCache(properties)); + new HiveFileListingCache(properties)); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .partitionKeyNames(Collections.singletonList("pt")).location("s3://bucket/") .inputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat") diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 6ea733a707d051..941c153f53190b 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -152,7 +152,7 @@ enum THdfsFileFormat { LZO_TEXT = 1, RC_FILE = 2, SEQUENCE_FILE =3, - AVRO = 4, + AVRO = 4, // Deprecated: reading this format is no longer supported. PARQUET = 5 } @@ -499,6 +499,7 @@ struct TTrinoConnectorTable { 3: optional map properties } +// Deprecated: the associated external table read support has been removed. struct TLakeSoulTable { 1: optional string db_name 2: optional string table_name @@ -538,7 +539,7 @@ struct TTableDescriptor { 20: optional TJdbcTable jdbcTable 21: optional TMCTable mcTable 22: optional TTrinoConnectorTable trinoConnectorTable - 23: optional TLakeSoulTable lakesoulTable + 23: optional TLakeSoulTable lakesoulTable // Deprecated: reading this format is no longer supported. 24: optional TDictionaryTable dictionaryTable 25: optional TRemoteDorisTable remoteDorisTable } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 4a84b7ed955900..cd02c8127ec903 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -124,7 +124,7 @@ enum TFileFormatType { FORMAT_JSON = 9, FORMAT_PROTO = 10, FORMAT_JNI = 11, - FORMAT_AVRO = 12, + FORMAT_AVRO = 12, // Deprecated: reading this format is no longer supported. FORMAT_CSV_LZ4BLOCK = 13, FORMAT_CSV_SNAPPYBLOCK = 14, FORMAT_WAL = 15, @@ -448,6 +448,7 @@ struct THudiFileDesc { 12: optional i64 schema_id; // for schema change. (native reader) } +// Deprecated: the associated external table read support has been removed. struct TLakeSoulFileDesc { 1: optional list file_paths; 2: optional list primary_keys; @@ -456,11 +457,13 @@ struct TLakeSoulFileDesc { 5: optional string options; } +// Deprecated: the associated external table read support has been removed. struct TTransactionalHiveDeleteDeltaDesc { 1: optional string directory_location 2: optional list file_names } +// Deprecated: the associated external table read support has been removed. struct TTransactionalHiveDesc { 1: optional string partition 2: optional list delete_deltas @@ -489,10 +492,10 @@ struct TTableFormatFileDesc { 2: optional TIcebergFileDesc iceberg_params 3: optional THudiFileDesc hudi_params 4: optional TPaimonFileDesc paimon_params - 5: optional TTransactionalHiveDesc transactional_hive_params + 5: optional TTransactionalHiveDesc transactional_hive_params // Deprecated: reading this format is no longer supported. 6: optional TMaxComputeFileDesc max_compute_params 7: optional TTrinoConnectorFileDesc trino_connector_params - 8: optional TLakeSoulFileDesc lakesoul_params + 8: optional TLakeSoulFileDesc lakesoul_params // Deprecated: reading this format is no longer supported. 9: optional i64 table_level_row_count = -1 10: optional TRemoteDorisFileDesc remote_doris_params // JDBC connection parameters (used when table_format_type == "jdbc") diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index e18c66ec89125a..ff06f95d7af46c 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -556,7 +556,7 @@ enum TTableType { JDBC_TABLE = 10, TEST_EXTERNAL_TABLE = 11, MAX_COMPUTE_TABLE = 12, - LAKESOUL_TABLE = 13, + LAKESOUL_TABLE = 13, // Deprecated: reading this format is no longer supported. TRINO_CONNECTOR_TABLE = 14, DICTIONARY_TABLE = 15, REMOTE_DORIS_TABLE = 16 diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 0baa216c1dced7..cb0a3a94d6f895 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -288,15 +288,6 @@ kerberosHdfsPort=8820 enableRefactorParamsHdfsTest=true -// LakeSoul catalog test config -enableLakesoulTest = false -lakesoulPGUser="*******" -lakesoulPGPwd="*******" -lakesoulPGUrl="*******" -lakesoulMinioAK="*******" -lakesoulMinioSK="*******" -lakesoulMinioEndpoint="*******" - // cloud metaServiceToken = "greedisgood9999" instanceId = "default_instance_id" diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out index 005f109060481b..207e787c850edd 100644 --- a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out +++ b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out @@ -1207,108 +1207,6 @@ true 127 1024 42 45097156608 3.1415 -2.713 \N \N \N \N \N 98 F W College 1000 Good 0 0 0 99 M U College 1000 Good 0 0 0 --- !test_32 -- -0 12 536870912 0 12 {"i":276, "j":605, "k":48} -0 12 536870912 1 12 {"i":700, "j":459, "k":443} -0 12 536870912 10 12 {"i":274, "j":96, "k":764} -0 12 536870912 11 12 {"i":808, "j":984, "k":271} -0 12 536870912 12 12 {"i":976, "j":552, "k":992} -0 12 536870912 13 12 {"i":17, "j":609, "k":884} -0 12 536870912 14 12 {"i":854, "j":406, "k":678} -0 12 536870912 15 12 {"i":332, "j":174, "k":239} -0 12 536870912 16 12 {"i":751, "j":656, "k":463} -0 12 536870912 17 12 {"i":170, "j":398, "k":265} -0 12 536870912 18 12 {"i":124, "j":776, "k":386} -0 12 536870912 19 12 {"i":715, "j":389, "k":836} -0 12 536870912 2 12 {"i":762, "j":18, "k":968} -0 12 536870912 20 12 {"i":235, "j":299, "k":140} -0 12 536870912 21 12 {"i":722, "j":872, "k":275} -0 12 536870912 22 12 {"i":332, "j":737, "k":315} -0 12 536870912 23 12 {"i":935, "j":844, "k":462} -0 12 536870912 24 12 {"i":214, "j":171, "k":728} -0 12 536870912 25 12 {"i":719, "j":134, "k":403} -0 12 536870912 26 12 {"i":920, "j":745, "k":973} -0 12 536870912 27 12 {"i":310, "j":338, "k":719} -0 12 536870912 28 12 {"i":71, "j":376, "k":619} -0 12 536870912 29 12 {"i":72, "j":899, "k":851} -0 12 536870912 3 12 {"i":810, "j":360, "k":994} -0 12 536870912 30 12 {"i":195, "j":106, "k":129} -0 12 536870912 31 12 {"i":105, "j":319, "k":601} -0 12 536870912 32 12 {"i":234, "j":729, "k":319} -0 12 536870912 33 12 {"i":262, "j":578, "k":371} -0 12 536870912 34 12 {"i":295, "j":158, "k":242} -0 12 536870912 35 12 {"i":850, "j":851, "k":584} -0 12 536870912 36 12 {"i":162, "j":61, "k":372} -0 12 536870912 37 12 {"i":50, "j":420, "k":929} -0 12 536870912 38 12 {"i":583, "j":305, "k":165} -0 12 536870912 39 12 {"i":439, "j":21, "k":659} -0 12 536870912 4 12 {"i":631, "j":299, "k":762} -0 12 536870912 40 12 {"i":892, "j":253, "k":781} -0 12 536870912 41 12 {"i":276, "j":220, "k":772} -0 12 536870912 42 12 {"i":940, "j":864, "k":359} -0 12 536870912 43 12 {"i":382, "j":431, "k":579} -0 12 536870912 44 12 {"i":560, "j":209, "k":104} -0 12 536870912 45 12 {"i":235, "j":386, "k":629} -0 12 536870912 46 12 {"i":924, "j":774, "k":135} -0 12 536870912 47 12 {"i":718, "j":285, "k":30} -0 12 536870912 48 12 {"i":960, "j":213, "k":835} -0 12 536870912 49 12 {"i":813, "j":660, "k":17} -0 12 536870912 5 12 {"i":24, "j":558, "k":124} -0 12 536870912 50 12 {"i":581, "j":303, "k":636} -0 12 536870912 51 12 {"i":774, "j":285, "k":179} -0 12 536870912 52 12 {"i":946, "j":912, "k":362} -0 12 536870912 53 12 {"i":709, "j":218, "k":70} -0 12 536870912 54 12 {"i":719, "j":507, "k":497} -0 12 536870912 55 12 {"i":5, "j":112, "k":882} -0 12 536870912 56 12 {"i":754, "j":608, "k":301} -0 12 536870912 57 12 {"i":980, "j":368, "k":697} -0 12 536870912 58 12 {"i":175, "j":664, "k":549} -0 12 536870912 59 12 {"i":599, "j":0, "k":663} -0 12 536870912 6 12 {"i":594, "j":30, "k":95} -0 12 536870912 60 12 {"i":920, "j":75, "k":943} -0 12 536870912 61 12 {"i":894, "j":5, "k":737} -0 12 536870912 62 12 {"i":745, "j":549, "k":359} -0 12 536870912 63 12 {"i":32, "j":523, "k":811} -0 12 536870912 64 12 {"i":465, "j":973, "k":171} -0 12 536870912 65 12 {"i":275, "j":762, "k":131} -0 12 536870912 66 12 {"i":758, "j":671, "k":206} -0 12 536870912 67 12 {"i":879, "j":572, "k":159} -0 12 536870912 68 12 {"i":9, "j":419, "k":400} -0 12 536870912 69 12 {"i":302, "j":534, "k":785} -0 12 536870912 7 12 {"i":782, "j":580, "k":536} -0 12 536870912 70 12 {"i":931, "j":378, "k":14} -0 12 536870912 71 12 {"i":201, "j":176, "k":326} -0 12 536870912 72 12 {"i":100, "j":697, "k":819} -0 12 536870912 73 12 {"i":245, "j":606, "k":265} -0 12 536870912 74 12 {"i":163, "j":13, "k":518} -0 12 536870912 75 12 {"i":952, "j":382, "k":262} -0 12 536870912 76 12 {"i":782, "j":6, "k":492} -0 12 536870912 77 12 {"i":494, "j":1, "k":22} -0 12 536870912 78 12 {"i":299, "j":930, "k":158} -0 12 536870912 79 12 {"i":7, "j":497, "k":514} -0 12 536870912 8 12 {"i":368, "j":225, "k":942} -0 12 536870912 80 12 {"i":560, "j":49, "k":609} -0 12 536870912 81 12 {"i":378, "j":761, "k":479} -0 12 536870912 82 12 {"i":268, "j":289, "k":733} -0 12 536870912 83 12 {"i":380, "j":601, "k":936} -0 12 536870912 84 12 {"i":722, "j":78, "k":87} -0 12 536870912 85 12 {"i":984, "j":742, "k":986} -0 12 536870912 86 12 {"i":414, "j":472, "k":265} -0 12 536870912 87 12 {"i":538, "j":342, "k":914} -0 12 536870912 88 12 {"i":881, "j":522, "k":464} -0 12 536870912 89 12 {"i":225, "j":732, "k":219} -0 12 536870912 9 12 {"i":916, "j":600, "k":70} -0 12 536870912 90 12 {"i":240, "j":594, "k":415} -0 12 536870912 91 12 {"i":33, "j":644, "k":707} -0 12 536870912 92 12 {"i":701, "j":128, "k":446} -0 12 536870912 93 12 {"i":532, "j":796, "k":318} -0 12 536870912 94 12 {"i":901, "j":401, "k":506} -0 12 536870912 95 12 {"i":164, "j":522, "k":425} -0 12 536870912 96 12 {"i":661, "j":302, "k":878} -0 12 536870912 97 12 {"i":854, "j":423, "k":597} -0 12 536870912 98 12 {"i":449, "j":961, "k":176} -0 12 536870912 99 12 {"i":614, "j":717, "k":177} - -- !test_34 -- 1 1a 10 10a diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 2af49b8fd63c6a..43f54b6ca921e1 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -212,9 +212,6 @@ kerberosHdfsPort=8820 enableNonCatalogKerberosTest = true -// LakeSoul catalog test config -enableLakesoulTest = true - // AWS iam role config // hudi p0 external regression test config diff --git a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy index 8c2a55d9e9dffa..2723cb431cc701 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy @@ -116,9 +116,6 @@ suite("test_hive_basic_type", "p0,external") { // At present, doris only supports three formats of orc parquet textfile, while others are not supported - // hive tables in avro format are not supported - //order_qt_34 """select * from ${catalog_name}.${ex_db_name}.avro_all_types limit 1;""" - // hive tables in SEQUENCEFILE format are not supported //order_qt_35 """select * from ${catalog_name}.${ex_db_name}.sequence_all_types limit 1;""" diff --git a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy index 7775781ea3bd7a..3c4a24cfe56d4f 100644 --- a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy +++ b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy @@ -215,12 +215,6 @@ suite("test_hdfs_orc_group0_orc_files", "p0,external") { "format" = "orc") order by _col0 limit 100; """ - uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/acid5k.orc" - order_qt_test_32 """ select * from HDFS( - "uri" = "${uri}", - "hadoop.username" = "${hdfsUserName}", - "format" = "orc") order by rowid limit 100; """ - uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/orc_index_int_string.orc" order_qt_test_34 """ select * from HDFS( "uri" = "${uri}", diff --git a/samples/datalake/lakesoul/README.md b/samples/datalake/lakesoul/README.md deleted file mode 100644 index 136c249fbe0c4f..00000000000000 --- a/samples/datalake/lakesoul/README.md +++ /dev/null @@ -1,277 +0,0 @@ -# doris-lakesoul-compose -## Launch Docker Compose - -First, we need to ensure the environmental parameters of the machine. - -``` -sysctl -w vm.max_map_count=2000000 -``` - -We can then start all the required containers via the script. - -``` -bash start_all.sh -``` -It will start a set of docker, the environment includes: -- doris -- postgreSQL -- mysql -- flink -- spark -- minIO - -And it will automatically create an lakesoul table. We can use these tables directly to experience doris. - -## LakeSoul Table QuickStart - -Now we can query this table through doris. - -``` -bash start_doris_client.sh -``` - -After entering the doris client, the lakesoul catalog has been created here, so the data of the lakesoul table can be directly queried. - -```sql -mysql> show catalogs; -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -| CatalogId | CatalogName | Type | IsCurrent | CreateTime | LastUpdateTime | Comment | -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -| 0 | internal | internal | Yes | NULL | NULL | Doris internal catalog | -| 10003 | lakesoul | lakesoul | No | 2024-11-08 05:47:59.304898431 | NULL | | -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -2 rows in set (0.04 sec) - -mysql> use lakesoul.tpch; -Reading table information for completion of table and column names -You can turn off this feature to get a quicker startup with -A - -Database changed - -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer_from_spark | -+---------------------+ -1 row in set (0.00 sec) - -mysql> select * from customer_from_spark where c_nationkey = 1 order by c_custkey limit 4; -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 14 | Customer#000000014 | KXkletMlL2JQEA | 1 | 11-845-129-3851 | 5266.30 | FURNITURE | , ironic packages across the unus | -| 30 | Customer#000000030 | nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY | 1 | 11-764-165-5076 | 9321.01 | BUILDING | lithely final requests. furiously unusual account | -| 59 | Customer#000000059 | zLOCP0wh92OtBihgspOGl4 | 1 | 11-355-584-3112 | 3458.60 | MACHINERY | ously final packages haggle blithely after the express deposits. furiou | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -4 rows in set (3.14 sec) - -mysql> select * from customer_from_spark where c_nationkey = 1 order by c_custkey desc limit 4; -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -| 14983 | Customer#000014983 | ERN3vq5Fvt4DL | 1 | 11-424-279-1846 | 841.22 | AUTOMOBILE | furiously slyly special foxes. express theodolites cajole carefully. special dinos haggle pinto | -| 14968 | Customer#000014968 | ,sykKTZBzVFl7ito1750v2TRYwmkRl2nvqGHwmx | 1 | 11-669-222-9657 | 6106.77 | HOUSEHOLD | ts above the furiously even deposits haggle across | -| 14961 | Customer#000014961 | JEIORcsBp6RpLYH 9gNdDyWJ | 1 | 11-490-251-5554 | 4006.35 | HOUSEHOLD | quests detect carefully final platelets! quickly final frays haggle slyly blithely final acc | -| 14940 | Customer#000014940 | bNoyCxPuqSwPLjbqjEUNGN d0mSP | 1 | 11-242-677-1085 | 8829.48 | HOUSEHOLD | ver the quickly express braids. regular dependencies haggle fluffily quickly i | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -4 rows in set (0.10 sec) -``` - -Doris can perform partition pruning on LakeSoul and speed up the query process through native reading. We can check this through `explain verbose`. - -```sql -mysql> explain verbose select * from customer_from_spark where c_nationkey < 3; -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Explain String(Old Planner) | -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS: | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_custkey` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_name` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_address` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_nationkey` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_phone` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_acctbal` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_mktsegment` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_comment` | -| PARTITION: UNPARTITIONED | -| | -| HAS_COLO_PLAN_NODE: false | -| | -| VRESULT SINK | -| MYSQL_PROTOCOL | -| | -| 1:VEXCHANGE | -| offset: 0 | -| tuple ids: 0 | -| | -| PLAN FRAGMENT 1 | -| | -| PARTITION: RANDOM | -| | -| HAS_COLO_PLAN_NODE: false | -| | -| STREAM DATA SINK | -| EXCHANGE ID: 01 | -| UNPARTITIONED | -| | -| 0:VplanNodeName | -| table: customer_from_spark | -| predicates: (`c_nationkey` < 3) | -| inputSplitNum=12, totalFileSize=0, scanRanges=12 | -| partition=0/0 | -| backends: | -| 10002 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00000-0568c817-d6bc-4fa1-bb9e-b311069b131c_00000.c000.parquet start: 0 length: 0 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00001-d99a8fe6-61ab-4285-94da-2f84f8746a8a_00001.c000.parquet start: 0 length: 0 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00002-8a8e396f-685f-4b0f-87fa-e2a3fe5be87e_00002.c000.parquet start: 0 length: 0 | -| ... other 8 files ... | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=0/part-00003-d5b598cd-5bed-412c-a26f-bb4bc9c937bc_00003.c000.parquet start: 0 length: 0 | -| numNodes=1 | -| pushdown agg=NONE | -| tuple ids: 0 | -| | -| Tuples: | -| TupleDescriptor{id=0, tbl=customer_from_spark} | -| SlotDescriptor{id=0, col=c_custkey, colUniqueId=0, type=int, nullable=false, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=1, col=c_name, colUniqueId=1, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=2, col=c_address, colUniqueId=2, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=3, col=c_nationkey, colUniqueId=3, type=int, nullable=false, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=4, col=c_phone, colUniqueId=4, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=5, col=c_acctbal, colUniqueId=5, type=decimalv3(15,2), nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=6, col=c_mktsegment, colUniqueId=6, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=7, col=c_comment, colUniqueId=7, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -57 rows in set (0.03 sec) - -``` - -## LakeSoul CDC Table - -Launch Flink CDC Job to sync mysql table. The mysql table is loaded when launching docker compose. - -``` -bash start_flink_cdc_job.sh -``` - -```sql -Start flink-cdc job... -SLF4J: Class path contains multiple SLF4J bindings. -SLF4J: Found binding in [jar:file:/opt/flink/lib/log4j-slf4j-impl-2.17.1.jar!/org/slf4j/impl/StaticLoggerBinder.class] -SLF4J: Found binding in [jar:file:/opt/hadoop/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class] -SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. -SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] -Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary. -Job has been submitted with JobID d1b3641dcd1ad85c6b373d49b1867e68 - -``` - - -Flink CDC Job will be launched. We can check the process of launching at `doris client` by recreate the lakesoul catalog. After the Flink CDC Job has been launched, we can see the syncing LakeSoul CDC table at `doris client`. - -```sql -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer_from_spark | -+---------------------+ -2 rows in set (0.00 sec) - - -mysql> drop catalog if exists lakesoul; -Query OK, 0 rows affected (0.00 sec) - -mysql> create catalog `lakesoul` properties ('type'='lakesoul', 'lakesoul.pg.username'='lakesoul_test', 'lakesoul.pg.password'='lakesoul_test', 'lakesoul.pg.url'='jdbc:postgresql://lakesoul-meta-pg:5432/lakesoul_test?stringtype=unspecified', 'minio.endpoint'='http://minio:9000', 'minio.access_key'='admin', 'minio.secret_key'='password'); -Query OK, 0 rows affected (0.01 sec) - -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer | -| customer_from_spark | -+---------------------+ -2 rows in set (0.00 sec) - -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 711.56 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -9 rows in set (1.09 sec) - -``` - -Enter the `mysql client` and try to modify data. - -``` -bash start_mysql_client.sh -``` - -Try update row from `mysql client`. - -```sql -mysql> update customer set c_acctbal=2211.26 where c_custkey=1; -Query OK, 1 row affected (0.01 sec) -Rows matched: 1 Changed: 1 Warnings: 0 -``` - -Back to `doris client` and check the data changing. - -```sql -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 2211.26 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -9 rows in set (0.11 sec) - -``` - -Try delete row from `mysql client`. - -```sql -mysql> delete from customer where c_custkey = 2; -Query OK, 1 row affected (0.01 sec) -``` - -Back to `doris client` and check the data changing. - -```sql -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 2211.26 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -8 rows in set (0.11 sec) - -``` diff --git a/samples/datalake/lakesoul/conf/my.conf b/samples/datalake/lakesoul/conf/my.conf deleted file mode 100644 index 8a4f2b1acd6edd..00000000000000 --- a/samples/datalake/lakesoul/conf/my.conf +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -[mysqld] -pid-file = /var/run/mysqld/mysqld.pid -socket = /var/run/mysqld/mysqld.sock -datadir = /var/lib/mysql -secure-file-priv= /data/tpch/ -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 - -# Custom config should go here -!includedir /etc/mysql/conf.d/ \ No newline at end of file diff --git a/samples/datalake/lakesoul/conf/properties b/samples/datalake/lakesoul/conf/properties deleted file mode 100644 index dceb3d5d11e5e8..00000000000000 --- a/samples/datalake/lakesoul/conf/properties +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# table_num 生成表数据量 -table_num=10 -# host mysql域名 -host=localhost -# user: mysql账号; password: mysql密码; port: mysql端口号; db: mysql数据库名; -user=root -password=root -port=3306 -db=test_cdc -timezone=UTC -# row_num随机向表中插入多少条数据 -row_num=500 -# delete_num随机删除多少条数据 -delete_num=50 diff --git a/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template b/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template deleted file mode 100644 index 385b2e772d2c80..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - FAIR - 1 - 2 - - - FIFO - 2 - 3 - - diff --git a/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template b/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template deleted file mode 100644 index ab96e03baed203..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template +++ /dev/null @@ -1,69 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the console -rootLogger.level = info -rootLogger.appenderRef.stdout.ref = console - -# In the pattern layout configuration below, we specify an explicit `%ex` conversion -# pattern for logging Throwables. If this was omitted, then (by default) Log4J would -# implicitly add an `%xEx` conversion pattern which logs stacktraces with additional -# class packaging information. That extra information can sometimes add a substantial -# performance overhead, so we disable it in our default logging config. -# For more information, see SPARK-39361. -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex - -# Set the default spark-shell/spark-sql log level to WARN. When running the -# spark-shell/spark-sql, the log level for these classes is used to overwrite -# the root logger's log level, so that the user can have different defaults -# for the shell and regular Spark apps. -logger.repl.name = org.apache.spark.repl.Main -logger.repl.level = warn - -logger.thriftserver.name = org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver -logger.thriftserver.level = warn - -# Settings to quiet third party logs that are too verbose -logger.jetty1.name = org.sparkproject.jetty -logger.jetty1.level = warn -logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle -logger.jetty2.level = error -logger.replexprTyper.name = org.apache.spark.repl.SparkIMain$exprTyper -logger.replexprTyper.level = info -logger.replSparkILoopInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter -logger.replSparkILoopInterpreter.level = info -logger.parquet1.name = org.apache.parquet -logger.parquet1.level = error -logger.parquet2.name = parquet -logger.parquet2.level = error - -# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support -logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler -logger.RetryingHMSHandler.level = fatal -logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry -logger.FunctionRegistry.level = error - -# For deploying Spark ThriftServer -# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 -appender.console.filter.1.type = RegexFilter -appender.console.filter.1.regex = .*Thrift error occurred during processing of message.* -appender.console.filter.1.onMatch = deny -appender.console.filter.1.onMismatch = neutral diff --git a/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template b/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template deleted file mode 100644 index f52d33fd642230..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template +++ /dev/null @@ -1,210 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# syntax: [instance].sink|source.[name].[options]=[value] - -# This file configures Spark's internal metrics system. The metrics system is -# divided into instances which correspond to internal components. -# Each instance can be configured to report its metrics to one or more sinks. -# Accepted values for [instance] are "master", "worker", "executor", "driver", -# and "applications". A wildcard "*" can be used as an instance name, in -# which case all instances will inherit the supplied property. -# -# Within an instance, a "source" specifies a particular set of grouped metrics. -# there are two kinds of sources: -# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will -# collect a Spark component's internal state. Each instance is paired with a -# Spark source that is added automatically. -# 2. Common sources, like JvmSource, which will collect low level state. -# These can be added through configuration options and are then loaded -# using reflection. -# -# A "sink" specifies where metrics are delivered to. Each instance can be -# assigned one or more sinks. -# -# The sink|source field specifies whether the property relates to a sink or -# source. -# -# The [name] field specifies the name of source or sink. -# -# The [options] field is the specific property of this source or sink. The -# source or sink is responsible for parsing this property. -# -# Notes: -# 1. To add a new sink, set the "class" option to a fully qualified class -# name (see examples below). -# 2. Some sinks involve a polling period. The minimum allowed polling period -# is 1 second. -# 3. Wildcard properties can be overridden by more specific properties. -# For example, master.sink.console.period takes precedence over -# *.sink.console.period. -# 4. A metrics specific configuration -# "spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties" should be -# added to Java properties using -Dspark.metrics.conf=xxx if you want to -# customize metrics system. You can also put the file in ${SPARK_HOME}/conf -# and it will be loaded automatically. -# 5. The MetricsServlet sink is added by default as a sink in the master, -# worker and driver, and you can send HTTP requests to the "/metrics/json" -# endpoint to get a snapshot of all the registered metrics in JSON format. -# For master, requests to the "/metrics/master/json" and -# "/metrics/applications/json" endpoints can be sent separately to get -# metrics snapshots of the master instance and applications. This -# MetricsServlet does not have to be configured. -# 6. The metrics system can also be configured using Spark configuration -# parameters. The relevant parameter names are formed by adding the -# prefix "spark.metrics.conf." to the configuration entries detailed in -# this file (see examples below). - -## List of available common sources and their properties. - -# org.apache.spark.metrics.source.JvmSource -# Note: Currently, JvmSource is the only available common source. -# It can be added to an instance by setting the "class" option to its -# fully qualified class name (see examples below). - -## List of available sinks and their properties. - -# org.apache.spark.metrics.sink.ConsoleSink -# Name: Default: Description: -# period 10 Poll period -# unit seconds Unit of the poll period - -# org.apache.spark.metrics.sink.CSVSink -# Name: Default: Description: -# period 10 Poll period -# unit seconds Unit of the poll period -# directory /tmp Where to store CSV files - -# org.apache.spark.metrics.sink.GangliaSink -# Name: Default: Description: -# host NONE Hostname or multicast group of the Ganglia server, -# must be set -# port NONE Port of the Ganglia server(s), must be set -# period 10 Poll period -# unit seconds Unit of the poll period -# ttl 1 TTL of messages sent by Ganglia -# dmax 0 Lifetime in seconds of metrics (0 never expired) -# mode multicast Ganglia network mode ('unicast' or 'multicast') - -# org.apache.spark.metrics.sink.JmxSink - -# org.apache.spark.metrics.sink.MetricsServlet -# Name: Default: Description: -# path VARIES* Path prefix from the web server root -# sample false Whether to show entire set of samples for histograms -# ('false' or 'true') -# -# * Default path is /metrics/json for all instances except the master. The -# master has two paths: -# /metrics/applications/json # App information -# /metrics/master/json # Master information - -# org.apache.spark.metrics.sink.PrometheusServlet -# Name: Default: Description: -# path VARIES* Path prefix from the web server root -# -# * Default path is /metrics/prometheus for all instances except the master. The -# master has two paths: -# /metrics/applications/prometheus # App information -# /metrics/master/prometheus # Master information - -# org.apache.spark.metrics.sink.GraphiteSink -# Name: Default: Description: -# host NONE Hostname of the Graphite server, must be set -# port NONE Port of the Graphite server, must be set -# period 10 Poll period -# unit seconds Unit of the poll period -# prefix EMPTY STRING Prefix to prepend to every metric's name -# protocol tcp Protocol ("tcp" or "udp") to use -# regex NONE Optional filter to send only metrics matching this regex string - -# org.apache.spark.metrics.sink.StatsdSink -# Name: Default: Description: -# host 127.0.0.1 Hostname or IP of StatsD server -# port 8125 Port of StatsD server -# period 10 Poll period -# unit seconds Units of poll period -# prefix EMPTY STRING Prefix to prepend to metric name - -## Examples -# Enable JmxSink for all instances by class name -#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink - -# Enable ConsoleSink for all instances by class name -#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink - -# Enable StatsdSink for all instances by class name -#*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink -#*.sink.statsd.prefix=spark - -# Polling period for the ConsoleSink -#*.sink.console.period=10 -# Unit of the polling period for the ConsoleSink -#*.sink.console.unit=seconds - -# Polling period for the ConsoleSink specific for the master instance -#master.sink.console.period=15 -# Unit of the polling period for the ConsoleSink specific for the master -# instance -#master.sink.console.unit=seconds - -# Enable CsvSink for all instances by class name -#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink - -# Polling period for the CsvSink -#*.sink.csv.period=1 -# Unit of the polling period for the CsvSink -#*.sink.csv.unit=minutes - -# Polling directory for CsvSink -#*.sink.csv.directory=/tmp/ - -# Polling period for the CsvSink specific for the worker instance -#worker.sink.csv.period=10 -# Unit of the polling period for the CsvSink specific for the worker instance -#worker.sink.csv.unit=minutes - -# Enable Slf4jSink for all instances by class name -#*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink - -# Polling period for the Slf4JSink -#*.sink.slf4j.period=1 -# Unit of the polling period for the Slf4jSink -#*.sink.slf4j.unit=minutes - -# Example configuration for Graphite sink -#*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink -#*.sink.graphite.host= -#*.sink.graphite.port= -#*.sink.graphite.period=10 -#*.sink.graphite.unit=seconds -#*.sink.graphite.prefix= - -# Enable JvmSource for instance master, worker, driver and executor -#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -# Example configuration for PrometheusServlet -#*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet -#*.sink.prometheusServlet.path=/metrics/prometheus -#master.sink.prometheusServlet.path=/metrics/master/prometheus -#applications.sink.prometheusServlet.path=/metrics/applications/prometheus diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf b/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf deleted file mode 100755 index 844709658bcde0..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Default system properties included when running spark-submit. -# This is useful for setting default environmental settings. - -# Example: -spark.sql.extensions com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension -spark.sql.catalog.lakesoul org.apache.spark.sql.lakesoul.catalog.LakeSoulCatalog -# spark.sql.defaultCatalog lakesoul -# spark.eventLog.enabled true -# spark.sql.catalogImplementation in-memory diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template b/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template deleted file mode 100644 index 19cba6e71ed190..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Default system properties included when running spark-submit. -# This is useful for setting default environmental settings. - -# Example: -# spark.master spark://master:7077 -# spark.eventLog.enabled true -# spark.eventLog.dir hdfs://namenode:8021/directory -# spark.serializer org.apache.spark.serializer.KryoSerializer -# spark.driver.memory 5g -# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template b/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template deleted file mode 100755 index e9491995e727eb..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file is sourced when running various Spark programs. -# Copy it as spark-env.sh and edit that to configure Spark for your site. - -# Options read when launching programs locally with -# ./bin/run-example or ./bin/spark-submit -# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files -# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node -# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program - -# Options read by executors and drivers running inside the cluster -# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node -# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program -# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data -# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos - -# Options read in any mode -# - SPARK_CONF_DIR, Alternate conf dir. (Default: ${SPARK_HOME}/conf) -# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). -# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) -# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) - -# Options read in any cluster manager using HDFS -# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files - -# Options read in YARN client/cluster mode -# - YARN_CONF_DIR, to point Spark towards YARN configuration files when you use YARN - -# Options for the daemons used in the standalone deploy mode -# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname -# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master -# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") -# - SPARK_WORKER_CORES, to set the number of cores to use on this machine -# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) -# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker -# - SPARK_WORKER_DIR, to set the working directory of worker processes -# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") -# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). -# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") -# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") -# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") -# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons -# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers - -# Options for launcher -# - SPARK_LAUNCHER_OPTS, to set config properties and Java options for the launcher (e.g. "-Dx=y") - -# Generic options for the daemons used in the standalone deploy mode -# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) -# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) -# - SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5. -# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) -# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) -# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) -# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. -# Options for native BLAS, like Intel MKL, OpenBLAS, and so on. -# You might get better performance to enable these options if using native BLAS (see SPARK-21305). -# - MKL_NUM_THREADS=1 Disable multi-threading of Intel MKL -# - OPENBLAS_NUM_THREADS=1 Disable multi-threading of OpenBLAS - -# Options for beeline -# - SPARK_BEELINE_OPTS, to set config properties only for the beeline cli (e.g. "-Dx=y") -# - SPARK_BEELINE_MEMORY, Memory for beeline (e.g. 1000M, 2G) (Default: 1G) diff --git a/samples/datalake/lakesoul/data/spark-conf/workers.template b/samples/datalake/lakesoul/data/spark-conf/workers.template deleted file mode 100644 index be42a638230b78..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/workers.template +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# A Spark Worker will be started on each of the machines listed below. -localhost \ No newline at end of file diff --git a/samples/datalake/lakesoul/docker-compose.env b/samples/datalake/lakesoul/docker-compose.env deleted file mode 100644 index 9485df04ae20a5..00000000000000 --- a/samples/datalake/lakesoul/docker-compose.env +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -PG_PORT=29013 -MYSQL_PORT=29014 -MINIO_UI_PORT=29002 -MINIO_API_PORT=29001 -DORIS_QUERY_PORT=29031 -FLINK_JOBMANAGER_PORT=28081 -LAKESOUL_META_HOST=lakesoul-meta-pg \ No newline at end of file diff --git a/samples/datalake/lakesoul/docker-compose.yml b/samples/datalake/lakesoul/docker-compose.yml deleted file mode 100644 index affe98e3fe6c17..00000000000000 --- a/samples/datalake/lakesoul/docker-compose.yml +++ /dev/null @@ -1,214 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3' - -services: - postgres: - image: postgres:14.5 - container_name: doris-lakesoul-pg - hostname: ${LAKESOUL_META_HOST} - networks: - - doris-lakesoul - ports: - - "${PG_PORT}:5432" - restart: always - environment: - POSTGRES_PASSWORD: lakesoul_test - POSTGRES_USER: lakesoul_test - POSTGRES_DB: lakesoul_test - command: - - "postgres" - - "-c" - - "max_connections=4096" - - "-c" - - "default_transaction_isolation=serializable" - volumes: - - ./sql/meta_init.sql:/docker-entrypoint-initdb.d/meta_init.sql - - ./sql/meta_cleanup.sql:/meta_cleanup.sql - - doris: - image: mysql:8.0.18 - container_name: doris-lakesoul-doris - networks: - - doris-lakesoul - hostname: doris - user: root - privileged: true - ports: - - ${DORIS_QUERY_PORT}:9030 - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - LD_LIBRARY_PATH=/opt/doris/be/lib - - JAVA_HOME=/opt/jdk-17 - volumes: - - ./packages/jdk-17:/opt/jdk-17 - - ./packages/doris-bin:/opt/doris-bin - - ./sql/init_doris.sql:/opt/doris-bin/init_doris.sql - - ./scripts:/opt/scripts - command: bin/bash /opt/scripts/start_doris.sh - - mysql: - image: mysql:8.0.18 - restart: always - container_name: doris-lakesoul-mysql-source - hostname: mysql - user: root - networks: - - doris-lakesoul - ports: - - ${MYSQL_PORT}:3306 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: tpch - command: - --max_connections=4096 - volumes: - - ./sql/mysql_init.sql:/docker-entrypoint-initdb.d/mysql_init.sql - - ./scripts:/opt/scripts - - ./conf/properties:/properties - - ./conf/my.conf:/etc/mysql/my.cnf - - ./packages/tpch-dbgen/customer.tbl:/data/tpch/customer.tbl - - minio: - image: bitnami/minio:latest - container_name: doris-lakesoul-minio - ports: - - ${MINIO_API_PORT}:9000 - - ${MINIO_UI_PORT}:9001 - environment: - MINIO_DEFAULT_BUCKETS: lakesoul-test-bucket:public - MINIO_ROOT_USER: admin - MINIO_ROOT_PASSWORD: password - MINIO_DOMAIN: minio - healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] - interval: 3s - timeout: 5s - retries: 3 - hostname: minio - networks: - - doris-lakesoul - # profiles: ["s3"] - - jobmanager: - image: flink:1.17.1-scala_2.12-java8 - container_name: doris-lakesoul-jobmanager - ports: - - ${FLINK_JOBMANAGER_PORT}:8081 - command: jobmanager - networks: - - doris-lakesoul - depends_on: - - postgres - - minio - deploy: - replicas: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.memory.process.size: 3000m - taskmanager.memory.flink.size: 2048m - s3.endpoint: http://minio:9000 - s3.access-key: admin - s3.secret-key: password - s3.path.style.access: true - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - - FLINK_HOME=/opt/flink - - HADOOP_HOME=/opt/hadoop - - HADOOP_CLASSPATH="/opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/yarn:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*" - volumes: - - ./work-dir/:/opt/flink/work-dir/ - - ./packages/hadoop-3.3.5:/opt/hadoop - - ./packages/jars/lakesoul-flink-1.17-2.6.1.jar:/opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar - - ./packages/jars/flink-s3-fs-hadoop-1.17.1.jar:/opt/flink/plugins/s3/flink-s3-fs-hadoop-1.17.1.jar - - ./packages/jars/flink-parquet-1.17.1.jar:/opt/flink/lib/flink-parquet-1.17.1.jar - - ./packages/jars/parquet-hadoop-bundle-1.12.3.jar:/opt/flink/lib/parquet-hadoop-bundle-1.12.3.jar - - ./sql/init_flink_tables.sql:/opt/flink/sql/init_tables.sql - - - taskmanager: - image: flink:1.17.1-scala_2.12-java8 - depends_on: - - jobmanager - command: taskmanager - deploy: - replicas: 2 - networks: - - doris-lakesoul - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.memory.process.size: 3000m - taskmanager.memory.flink.size: 2048m - taskmanager.numberOfTaskSlots: 6 - s3.endpoint: http://minio:9000 - s3.access-key: admin - s3.secret-key: password - s3.path.style.access: true - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - - FLINK_HOME=/opt/flink - - HADOOP_HOME=/opt/hadoop - - HADOOP_CLASSPATH="/opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/yarn:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*" - volumes: - - ./work-dir/:/opt/flink/work-dir/ - - ./packages/hadoop-3.3.5:/opt/hadoop - - ./packages/jars/lakesoul-flink-1.17-2.6.1.jar:/opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar - - ./packages/jars/flink-s3-fs-hadoop-1.17.1.jar:/opt/flink/plugins/s3/flink-s3-fs-hadoop-1.17.1.jar - - ./packages/jars/flink-parquet-1.17.1.jar:/opt/flink/lib/flink-parquet-1.17.1.jar - - ./packages/jars/parquet-hadoop-bundle-1.12.3.jar:/opt/flink/lib/parquet-hadoop-bundle-1.12.3.jar - - spark: - image: bitnami/spark:3.3.1 - container_name: doris-lakesoul-spark - build: spark/ - privileged: true - user: root - volumes: - - ./packages/jars/lakesoul-spark-3.3-2.6.1.jar:/opt/bitnami/spark/jars/lakesoul-spark-3.3-2.6.1.jar - - ./data/spark-conf:/opt/bitnami/spark/conf - - ./sql/prepare_data.sql:/opt/sql/prepare_data.sql - - ./packages/tpch-dbgen/customer.tbl:/data/tpch/customer.tbl - - ./packages/tpch-dbgen/orders.tbl:/data/tpch/orders.tbl - depends_on: - - postgres - - minio - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - networks: - - doris-lakesoul - -networks: - doris-lakesoul: - ipam: - driver: default \ No newline at end of file diff --git a/samples/datalake/lakesoul/scripts/load_tpch_data.sh b/samples/datalake/lakesoul/scripts/load_tpch_data.sh deleted file mode 100644 index bd1534d0912a2f..00000000000000 --- a/samples/datalake/lakesoul/scripts/load_tpch_data.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -git clone https://github.com/databricks/tpch-dbgen.git -cd tpch-dbgen -make -./dbgen -f -s 0.1 \ No newline at end of file diff --git a/samples/datalake/lakesoul/scripts/start_doris.sh b/samples/datalake/lakesoul/scripts/start_doris.sh deleted file mode 100644 index ef82cc0e7dfd88..00000000000000 --- a/samples/datalake/lakesoul/scripts/start_doris.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export JAVA_HOME=/opt/jdk-17 -export PATH=${JAVA_HOME}/bin:${PATH} - -echo 'start to copy...' -cp -r /opt/doris-bin /opt/doris - -echo 'start fe...' -rm -rf /opt/doris/fe/doris-meta/* -/opt/doris/fe/bin/start_fe.sh --daemon - -echo 'start be...' -rm -rf /opt/doris/be/storage/* -/opt/doris/be/bin/start_be.sh --daemon - -while [[ ! -f "/opt/doris/fe/log/fe.log" ]]; do - echo "wait log..." - sleep 2 -done - -QE=$(grep "QE service start." /opt/doris/fe/log/fe.log) -while [[ -z "${QE}" ]]; do - echo "wait fe..." - sleep 2 - QE=$(grep "QE service start." /opt/doris/fe/log/fe.log) -done - -echo 'doris is started' - -MYSQL_ERROR=$(mysql -u root -P 9030 -h doris &1) -ERR=$(echo "${MYSQL_ERROR}" | grep "Can't connect to MySQL") -echo "${ERR}" -while [[ -n "${ERR}" ]]; do - echo "wait mysql..." - sleep 2 - MYSQL_ERROR=$(mysql -u root -P 9030 -h doris &1) - ERR=$(echo "${MYSQL_ERROR}" | grep "Can't connect to MySQL") -done - -echo 'doris is inited' - -tail -F /dev/null diff --git a/samples/datalake/lakesoul/sql/init_doris.sql b/samples/datalake/lakesoul/sql/init_doris.sql deleted file mode 100644 index ead15ec80e5aca..00000000000000 --- a/samples/datalake/lakesoul/sql/init_doris.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER SYSTEM ADD BACKEND '127.0.0.1:9050'; - -drop catalog if exists lakesoul; - -create catalog `lakesoul` properties ( - 'type'='lakesoul', - 'lakesoul.pg.username'='lakesoul_test', - 'lakesoul.pg.password'='lakesoul_test', - 'lakesoul.pg.url'='jdbc:postgresql://lakesoul-meta-pg:5432/lakesoul_test?stringtype=unspecified', - 'minio.endpoint'='http://minio:9000', - 'minio.access_key'='admin', - 'minio.secret_key'='password' - ); \ No newline at end of file diff --git a/samples/datalake/lakesoul/sql/meta_cleanup.sql b/samples/datalake/lakesoul/sql/meta_cleanup.sql deleted file mode 100644 index dd25a7ee958879..00000000000000 --- a/samples/datalake/lakesoul/sql/meta_cleanup.sql +++ /dev/null @@ -1,11 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -delete from namespace; -insert into namespace(namespace, properties, comment) values ('default', '{}', ''); -delete from data_commit_info; -delete from table_info; -delete from table_path_id; -delete from table_name_id; -delete from partition_info; diff --git a/samples/datalake/lakesoul/sql/meta_init.sql b/samples/datalake/lakesoul/sql/meta_init.sql deleted file mode 100644 index 28b4eca4b6c164..00000000000000 --- a/samples/datalake/lakesoul/sql/meta_init.sql +++ /dev/null @@ -1,145 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -create table if not exists namespace -( - namespace text, - properties json, - comment text, - domain text default 'public', - primary key (namespace) -); - -insert into namespace(namespace, properties, comment) values ('default', '{}', '') -ON CONFLICT DO NOTHING; - -create table if not exists table_info -( - table_id text, - table_namespace text default 'default', - table_name text, - table_path text, - table_schema text, - properties json, - partitions text, - domain text default 'public', - primary key (table_id) -); - -create table if not exists table_name_id -( - table_name text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_name, table_namespace) -); - -create table if not exists table_path_id -( - table_path text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_path) -); - -DO -$$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'data_file_op') THEN - create type data_file_op as - ( - path text, - file_op text, - size bigint, - file_exist_cols text - ); - END IF; - END -$$; - -create table if not exists data_commit_info -( - table_id text, - partition_desc text, - commit_id UUID, - file_ops data_file_op[], - commit_op text, - committed boolean default 'false', - timestamp bigint, - domain text default 'public', - primary key (table_id, partition_desc, commit_id) -); - -create table if not exists partition_info -( - table_id text, - partition_desc text, - version int, - commit_op text, - timestamp bigint DEFAULT (date_part('epoch'::text, now()) * (1000)::double precision), - snapshot UUID[], - expression text, - domain text default 'public', - primary key (table_id, partition_desc, version) -); - -CREATE OR REPLACE FUNCTION partition_insert() RETURNS TRIGGER AS -$$ -DECLARE - rs_version integer; - rs_table_path text; - rs_table_namespace text; -BEGIN - if NEW.commit_op <> 'CompactionCommit' then - select version - INTO rs_version - from partition_info - where table_id = NEW.table_id - and partition_desc = NEW.partition_desc - and version != NEW.version - and commit_op = 'CompactionCommit' - order by version desc - limit 1; - if rs_version >= 0 then - if NEW.version - rs_version >= 3 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - else - if NEW.version >= 2 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - end if; - RETURN NULL; - end if; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER partition_table_change - AFTER INSERT - ON partition_info - FOR EACH ROW -EXECUTE PROCEDURE partition_insert(); - -create table if not exists global_config -( - key text, - value text, - primary key (key) -); - diff --git a/samples/datalake/lakesoul/sql/mysql_init.sql b/samples/datalake/lakesoul/sql/mysql_init.sql deleted file mode 100644 index d7fd3c01c5c817..00000000000000 --- a/samples/datalake/lakesoul/sql/mysql_init.sql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE TABLE customer ( - c_custkey INT NOT NULL, - c_name VARCHAR(25), - c_address VARCHAR(40), - c_nationkey INT, - c_phone CHAR(15), - c_acctbal DECIMAL(15,2), - c_mktsegment VARCHAR(10), - c_comment VARCHAR(117), - PRIMARY KEY (c_custkey) -); - -LOAD DATA INFILE '/data/tpch/customer.tbl' -INTO TABLE customer -FIELDS TERMINATED BY '|' -LINES TERMINATED BY '\n'; - -CREATE TABLE orders ( - o_orderkey INT NOT NULL, - o_custkey INT, - o_orderstatus CHAR(1), - o_totalprice DECIMAL(15,2), - o_orderdate DATE, - o_orderpriority CHAR(15), - o_clerk CHAR(15), - o_shippriority INT, - o_comment VARCHAR(79), - PRIMARY KEY (o_orderkey), - FOREIGN KEY (o_custkey) REFERENCES customer(c_custkey) -); - -LOAD DATA INFILE '/data/tpch/orders.tbl' -INTO TABLE orders -FIELDS TERMINATED BY '|' -LINES TERMINATED BY '\n'; \ No newline at end of file diff --git a/samples/datalake/lakesoul/sql/prepare_data.sql b/samples/datalake/lakesoul/sql/prepare_data.sql deleted file mode 100644 index a5b7de178b3f74..00000000000000 --- a/samples/datalake/lakesoul/sql/prepare_data.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE TABLE spark_catalog.default.customer ( - c_custkey INT NOT NULL, - c_name STRING, - c_address STRING, - c_nationkey INT, - c_phone STRING, - c_acctbal DECIMAL(15, 2), - c_mktsegment STRING, - c_comment STRING -) -USING csv -OPTIONS (path '/data/tpch/customer.tbl', delimiter '|'); - -USE lakesoul; -CREATE NAMESPACE IF NOT EXISTS tpch; - -CREATE TABLE IF NOT EXISTS tpch.customer_from_spark ( - c_custkey INT NOT NULL, - c_name STRING, - c_address STRING, - c_nationkey INT, - c_phone STRING, - c_acctbal DECIMAL(15, 2), - c_mktsegment STRING, - c_comment STRING) -USING lakesoul -PARTITIONED BY (c_nationkey) -LOCATION 's3://lakesoul-test-bucket/data/tpch/customer_from_spark' -TBLPROPERTIES ( - 'hashBucketNum' = '4', - 'hashPartitions' = 'c_custkey'); - - -INSERT OVERWRITE lakesoul.tpch.customer_from_spark -SELECT c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment, c_nationkey -FROM ( - SELECT COALESCE(c_custkey, -1) AS c_custkey, - c_name, c_address, c_phone, c_mktsegment, c_comment, c_acctbal, - COALESCE(CAST(c_nationkey AS int), -1) AS c_nationkey - FROM spark_catalog.default.customer -); diff --git a/samples/datalake/lakesoul/start_all.sh b/samples/datalake/lakesoul/start_all.sh deleted file mode 100644 index 760e4d659932df..00000000000000 --- a/samples/datalake/lakesoul/start_all.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# DORIS_PACKAGE=apache-doris-3.0.2-bin-x64 -# DORIS_DOWNLOAD_URL=https://apache-doris-releases.oss-accelerate.aliyuncs.com -DORIS_PACKAGE=apache-doris-3.0.2-lakesoul-bin-x64 -DORIS_DOWNLOAD_URL=https://lakesoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/doris -LAKESOUL_VERSION=2.6.1 -FLINK_LAKESOUL_JAR=lakesoul-flink-1.17-${LAKESOUL_VERSION}.jar -SPARK_LAKESOUL_JAR=lakesoul-spark-3.3-${LAKESOUL_VERSION}.jar - -download_source_file() { - local FILE_PATH="$1" - local EXPECTED_MD5="$2" - local DOWNLOAD_URL="$3" - - echo "solve for ${FILE_PATH} ..." - - if [[ -f "${FILE_PATH}" ]]; then - local FILE_MD5 - echo "compare md5sum ..." - FILE_MD5=$(md5sum "${FILE_PATH}" | awk '{ print $1 }') - - if [[ "${FILE_MD5}" = "${EXPECTED_MD5}" ]]; then - echo "${FILE_PATH} is ready!" - else - echo "${FILE_PATH} is broken, Redownloading ..." - rm "${FILE_PATH}" - wget "${DOWNLOAD_URL}"/"${FILE_PATH}" - fi - else - echo "downloading ${FILE_PATH} ...", - wget "${DOWNLOAD_URL}"/"${FILE_PATH}" - fi -} - -unpack_tar() { - local TAR_FILE_PATH="$1" - local OUT_DIR="$2" - - echo "unpack for ${TAR_FILE_PATH} ..." - - if [[ ! -f "${OUT_DIR}/SUCCESS" ]]; then - echo "Prepare ${OUT_DIR} environment" - if [[ -d "${OUT_DIR}" ]]; then - echo "Remove broken ${OUT_DIR}" - rm -rf ${OUT_DIR} - fi - echo "Unpackage ${TAR_FILE_PATH}" - tar xzf "${TAR_FILE_PATH}" - touch "${OUT_DIR}"/SUCCESS - fi -} - - -curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -cd "${curdir}" || exit - -if [[ ! -d "packages" ]]; then - mkdir packages -fi -cd packages || exit - -download_source_file "${DORIS_PACKAGE}.tar.gz" "b08d23cef21a34cee3a8ce337bed559f" "${DORIS_DOWNLOAD_URL}" - -download_source_file "${FLINK_LAKESOUL_JAR}" "de38f63ebd44835e9a37412908a0cdc0" "https://github.com/lakesoul-io/LakeSoul/releases/download/v${LAKESOUL_VERSION}" - -download_source_file "${SPARK_LAKESOUL_JAR}" "31a923958fb5398bbd00de073a88a787" "https://github.com/lakesoul-io/LakeSoul/releases/download/v${LAKESOUL_VERSION}" - -download_source_file "openjdk-17_linux-x64_bin.tar.gz" "5d1e9bc9be1570768485df4ff665821d" "https://mirrors.huaweicloud.com/openjdk/17/" -unpack_tar "openjdk-17_linux-x64_bin.tar.gz" "jdk-17" - -# download_source_file "hadoop-3.3.5.tar.gz" "1b6175712d813e8baec48ed68098ca85" "https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5" -download_source_file "hadoop-3.3.5.tar.gz" "1b6175712d813e8baec48ed68098ca85" "https://lakesoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/doris" -unpack_tar "hadoop-3.3.5.tar.gz" "hadoop-3.3.5" - - -download_source_file "flink-s3-fs-hadoop-1.17.1.jar" "0a631b07ba3e3b6c54e7c7c920ac6487" "https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.17.1" -download_source_file "parquet-hadoop-bundle-1.12.3.jar" "3a78d684a1938e68c6a57f59863e9106" "https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.12.3" -download_source_file "flink-parquet-1.17.1.jar" "559fda5535d4018fb923c4ec198340f0" "https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.17.1" - - -if [[ ! -f "doris-bin/SUCCESS" ]]; then - echo "Prepare ${DORIS_PACKAGE} environment" - if [[ -d "doris-bin" ]]; then - echo "Remove broken ${DORIS_PACKAGE}" - rm -rf doris-bin - fi - echo "Unpackage ${DORIS_PACKAGE}" - tar xzf "${DORIS_PACKAGE}".tar.gz - - mv "${DORIS_PACKAGE}" doris-bin - sed -i 's/-lt 2000000 ]]/-lt 60000 ]]/g' doris-bin/be/bin/start_be.sh - touch doris-bin/SUCCESS -fi - -if [[ ! -f "jars/SUCCESS" ]]; then - echo "Prepare jars environment" - if [[ -d "jars" ]]; then - echo "Remove broken jars" - rm -rf jars - fi - mkdir jars - cp ./*.jar jars/ - touch jars/SUCCESS -fi - -if [[ ! -f "tpch-dbgen/SUCCESS" ]]; then - echo "Load tpch data..." - if [[ -d "tpch-dbgen" ]]; then - echo "Remove broken tpch-dbgen" - rm -rf tpch-dbgen - fi - - git clone https://github.com/databricks/tpch-dbgen.git - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - - touch SUCCESS - cd ../ -fi - -cd ../ - -echo "Start docker-compose..." -sudo docker compose -f docker-compose.yml --env-file docker-compose.env up -d - -echo "Start prepare data for lakesoul tables..." -sudo docker exec -it doris-lakesoul-spark spark-sql --conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem --conf spark.hadoop.fs.s3a.buffer.dir=/opt/spark/work-dir/s3a --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider --conf spark.hadoop.fs.s3a.access.key=admin --conf spark.hadoop.fs.s3a.secret.key=password -f /opt/sql/prepare_data.sql | tee -a init.log >/dev/null - - -echo "=============================================================================" -echo "Success to launch doris+iceberg+paimon+flink+spark+minio environments!" -echo "You can:" -echo " 'bash start_doris_client.sh' to login into doris" -echo " 'bash start_flink_client.sh' to login into flink" -echo " 'bash start_spark_sql.sh' to login into spark" -echo " 'bash start_mysql_client.sh' to login into mysql for test" -echo "=============================================================================" diff --git a/samples/datalake/lakesoul/start_doris_client.sh b/samples/datalake/lakesoul/start_doris_client.sh deleted file mode 100644 index a411102b7978ed..00000000000000 --- a/samples/datalake/lakesoul/start_doris_client.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-doris sh -c "mysql -u root -h doris -P 9030" diff --git a/samples/datalake/lakesoul/start_flink_cdc_job.sh b/samples/datalake/lakesoul/start_flink_cdc_job.sh deleted file mode 100644 index cd20f1f1de7ffb..00000000000000 --- a/samples/datalake/lakesoul/start_flink_cdc_job.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -echo "Start flink-cdc job..." -sudo docker exec -t doris-lakesoul-jobmanager flink run -d -c org.apache.flink.lakesoul.entry.MysqlCdc /opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar --source_db.host mysql --source_db.port 3306 --source_db.db_name tpch --source_db.user root --source_db.password root --source.parallelism 2 --sink.parallelism 4 --use.cdc true --warehouse_path s3://lakesoul-test-bucket/data/ --flink.checkpoint s3://lakesoul-test-bucket/chk --flink.savepoint s3://lakesoul-test-bucket/svp --job.checkpoint_interval 5000 --server_time_zone UTC diff --git a/samples/datalake/lakesoul/start_flink_client.sh b/samples/datalake/lakesoul/start_flink_client.sh deleted file mode 100644 index e30634b5f89dfc..00000000000000 --- a/samples/datalake/lakesoul/start_flink_client.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-jobmanager sql-client.sh diff --git a/samples/datalake/lakesoul/start_mysql_client.sh b/samples/datalake/lakesoul/start_mysql_client.sh deleted file mode 100644 index e298343f488ac2..00000000000000 --- a/samples/datalake/lakesoul/start_mysql_client.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-mysql-source sh -c "mysql -u root -h mysql -p tpch -P 3306" diff --git a/samples/datalake/lakesoul/start_spark_sql.sh b/samples/datalake/lakesoul/start_spark_sql.sh deleted file mode 100644 index d6d777923f6f38..00000000000000 --- a/samples/datalake/lakesoul/start_spark_sql.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -docker exec -it doris-lakesoul-spark spark-sql --conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem --conf spark.hadoop.fs.s3a.buffer.dir=/opt/spark/work-dir/s3a --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider --conf spark.hadoop.fs.s3a.access.key=admin --conf spark.hadoop.fs.s3a.secret.key=password \ No newline at end of file diff --git a/samples/datalake/lakesoul/stop_all.sh b/samples/datalake/lakesoul/stop_all.sh deleted file mode 100644 index 63c244756078da..00000000000000 --- a/samples/datalake/lakesoul/stop_all.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker compose -f docker-compose.yml --env-file docker-compose.env down diff --git a/samples/datalake/lakesoul/work-dir/lakesoul.properties b/samples/datalake/lakesoul/work-dir/lakesoul.properties deleted file mode 100644 index a37fffa1ae40ef..00000000000000 --- a/samples/datalake/lakesoul/work-dir/lakesoul.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -lakesoul.pg.driver=com.lakesoul.shaded.org.postgresql.Driver -lakesoul.pg.url=jdbc:postgresql://doris-lakesoul-docker-compose-env-lakesoul-meta-db-1:5432/lakesoul_test?stringtype=unspecified -lakesoul.pg.username=lakesoul_test -lakesoul.pg.password=lakesoul_test From 31d7e91ccc09da051218d8bbd27166a2d138100e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 11:45:57 +0800 Subject: [PATCH 2/2] [fix](fe) Preserve retired ACID session variable compatibility ### What problem does this PR solve? Related PR: #69601 Problem Summary: Clients that still set skip_checking_acid_version_file when initializing a connection encounter an unknown-variable error after the Hive ACID reader removal. Register the retired name in the existing removed-variable compatibility set so SET remains a no-op and both variable-read paths use the empty-string fallback. Do not restore the session field or the removed Hive read support. ### Release note Continue accepting the retired skip_checking_acid_version_file session variable as a compatibility no-op. ### Check List (For Author) - Test: Targeted FE unit tests and FE Checkstyle. - Behavior changed: Yes, preserve legacy client session initialization. - Does this need documentation: No. --- .../java/org/apache/doris/qe/VariableMgr.java | 2 + .../qe/RemovedAcidSessionVariableTest.java | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 629378a73534d6..ac3a58e09adad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -157,6 +157,8 @@ public class VariableMgr { "enable_phrase_query_sequential_opt", "enable_rust_lance_reader", "enable_paimon_cpp_reader", + // Keep old connection-init SET statements valid after removing Hive ACID reads. + "skip_checking_acid_version_file", "enable_nereids_trace", "nereids_trace_event_mode", "shuffled_agg_node_ids", diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java new file mode 100644 index 00000000000000..507bfbeda68471 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.analysis.SetType; +import org.apache.doris.analysis.SetVar; +import org.apache.doris.analysis.VariableExpr; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.trees.expressions.literal.Literal; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +class RemovedAcidSessionVariableTest { + @Test + void removedAcidVariableRemainsANoOpForSessionInitialization() throws Exception { + SessionVariable variables = new SessionVariable(); + for (String name : new String[] {"skip_checking_acid_version_file", + "skip_checking_acid_version_file".toUpperCase(Locale.ROOT)}) { + for (boolean value : new boolean[] {true, false}) { + SetVar setVar = new SetVar(SetType.SESSION, name, new BoolLiteral(value)); + Assertions.assertDoesNotThrow(() -> VariableMgr.setVar(variables, setVar)); + Assertions.assertDoesNotThrow(() -> VariableMgr.setVarForNonMasterFE(variables, setVar)); + } + VariableExpr expression = new VariableExpr(name); + VariableMgr.fillValue(variables, expression); + Assertions.assertEquals(Type.VARCHAR, expression.getType()); + Assertions.assertEquals("", VariableMgr.getValue(variables, expression)); + Literal literal = VariableMgr.getLiteral(variables, name, SetType.SESSION); + Assertions.assertNotNull(literal); + Assertions.assertEquals("", literal.getStringValue()); + Assertions.assertNull(VariableMgr.getVarContext(name)); + } + Assertions.assertFalse(VariableMgr.toMap(variables).containsKey("skip_checking_acid_version_file")); + } +}