diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index e8666131ffbaca..7e6f2ad15791e7 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type_struct.h" #include "exec/operator/olap_scan_operator.h" #include "exec/operator/scan_operator.h" +#include "exec/scan/file_scan_range_utils.h" #include "exec/scan/file_scanner.h" #include "exec/scan/file_scanner_v2.h" #include "exec/scan/scanner_context.h" @@ -188,8 +189,8 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // ADBC only has a FileScannerV2 reader, and enable_file_scanner_v2 is a session variable marked // fuzzy=true, so the regression harness flips it to false at random. Without letting adbc // through unconditionally, those queries land in v1, which has no "adbc" branch, and come back - // with an undiagnosable NotSupported -- showing up in CI as a random failure. This is the same - // mechanism as is_transactional_hive below, pointed the other way. Loads are left alone: there + // with an undiagnosable NotSupported -- showing up in CI as a random failure. + // Loads are left alone: there // is no ADBC load path, so widening the rule to cover them would only route them somewhere they // still cannot run. // Fluss is in the same position, and reaches this the same way: FlussScanPlanProvider stamps the @@ -202,9 +203,6 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o scan_params.table_format_params.table_format_type == "fluss")) { return true; } - const bool is_transactional_hive = - scan_params.__isset.table_format_params && - scan_params.table_format_params.table_format_type == "transactional_hive"; // Hybrid scans advertise native Parquet at scan level because ranges can arrive after the // scanner is selected; FORMAT_JNI alone does not imply that every range uses JNI. const bool is_hybrid_native_parquet = scan_params.format_type == TFileFormatType::FORMAT_JNI && @@ -224,7 +222,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing // every file. return scanner_v2_requested && !is_load && - scan_params.format_type != TFileFormatType::FORMAT_LANCE && !is_transactional_hive; + scan_params.format_type != TFileFormatType::FORMAT_LANCE; } Status FileScanLocalState::_init_scanners(std::list* scanners) { @@ -247,6 +245,7 @@ Status FileScanLocalState::_init_scanners(std::list* scanners) { } else { scan_params = _split_source->get_params(); } + RETURN_IF_ERROR(validate_file_scan_support(*scan_params)); const bool is_load = state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr; // TODO: Use scanner v2 for all queries. diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 615b553990bfba..4f20f3e3dd8ddb 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -463,7 +463,6 @@ TFileRangeDesc RowIdStorageReader::build_external_fetch_range(const TFileRangeDe // needed by virtual columns, and do not mutate the FileMapping shared by other fetches. auto range = source_range; range.table_format_params.iceberg_params.__set_delete_files({}); - range.table_format_params.transactional_hive_params = TTransactionalHiveDesc {}; return range; } diff --git a/be/src/exec/scan/file_scan_range_utils.h b/be/src/exec/scan/file_scan_range_utils.h index cd4c06c953458b..9b9e91fc221441 100644 --- a/be/src/exec/scan/file_scan_range_utils.h +++ b/be/src/exec/scan/file_scan_range_utils.h @@ -17,10 +17,38 @@ #pragma once +#include "common/status.h" #include "gen_cpp/PlanNodes_types.h" namespace doris { +inline Status validate_file_scan_support(const TFileScanRangeParams& params, + const TFileRangeDesc* range = nullptr) { + // Deprecated table markers must never fall through to plain file reads: that would lose + // Hive transaction visibility and LakeSoul merge semantics, including in metadata counts. + auto validate_table = [](const TTableFormatFileDesc& table) -> Status { + if (table.table_format_type == "transactional_hive") { + return Status::NotSupported("Hive transactional table reads are no longer supported"); + } + if (table.table_format_type == "lakesoul") { + return Status::NotSupported("LakeSoul reads are no longer supported"); + } + return Status::OK(); + }; + if (params.__isset.table_format_params) { + RETURN_IF_ERROR(validate_table(params.table_format_params)); + } + if (range != nullptr && range->__isset.table_format_params) { + RETURN_IF_ERROR(validate_table(range->table_format_params)); + } + const auto format = range != nullptr && range->__isset.format_type ? range->format_type + : params.format_type; + if (format == TFileFormatType::FORMAT_AVRO) { + return Status::NotSupported("Avro file reads are no longer supported"); + } + return Status::OK(); +} + inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) { // Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a // missing data or delete file would silently return an incomplete or incorrect snapshot. diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 15ae816cad2586..ff210680576ece 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -80,7 +80,6 @@ #include "format/table/paimon_reader.h" #include "format/table/partition_column_filler.h" #include "format/table/remote_doris_reader.h" -#include "format/table/transactional_hive_reader.h" #include "format/table/trino_connector_jni_reader.h" #include "format/text/text_reader.h" #include "io/cache/block_file_cache_profile.h" @@ -152,6 +151,7 @@ void FileScanner::_configure_file_scan_handlers() { } Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { + RETURN_IF_ERROR(validate_file_scan_support(*_params)); RETURN_IF_ERROR(Scanner::init(state, conjuncts)); _get_block_timer = ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerGetBlockTime", 1); @@ -1030,6 +1030,7 @@ Status FileScanner::_get_next_reader() { } const TFileRangeDesc& range = _current_range; + RETURN_IF_ERROR(validate_file_scan_support(*_params, &range)); _current_range_path = range.path; if (!_partition_slot_index_map.empty()) { @@ -1456,20 +1457,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, octx.slot_id_to_filter_conjuncts = &_slot_id_to_filter_conjuncts; if (range.__isset.table_format_params && - range.table_format_params.table_format_type == "transactional_hive") { - // TransactionalHiveReader IS-A OrcReader, no wrapping needed - auto tran_orc_reader = TransactionalHiveReader::create_unique( - _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), - _io_ctx, file_meta_cache_ptr); - tran_orc_reader->set_create_row_id_column_iterator_func( - [this]() -> std::shared_ptr { - return _create_row_id_column_iterator(); - }); - init_status = static_cast(tran_orc_reader.get())->init_reader(&octx); - - _cur_reader = std::move(tran_orc_reader); - } else if (range.__isset.table_format_params && - range.table_format_params.table_format_type == "iceberg") { + range.table_format_params.table_format_type == "iceberg") { // IcebergOrcReader IS-A OrcReader (CRTP mixin), no wrapping needed std::unique_ptr iceberg_reader = IcebergOrcReader::create_unique( _kv_cache, _profile, _state, *_params, range, _state->batch_size(), @@ -1587,6 +1575,7 @@ Status FileScanner::_generate_truncate_columns(bool need_to_get_parsed_schema) { } Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) { + RETURN_IF_ERROR(validate_file_scan_support(*_params, &range)); _current_range = range; _file_cache_statistics.reset(new io::FileCacheStatistics()); diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 975f1a74e60517..b869533b38b641 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -376,6 +376,7 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile Status FileScannerV2::_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { + RETURN_IF_ERROR(validate_file_scan_support(params, &range)); if (!is_supported(params, range)) { return Status::NotSupported( "FileScannerV2 does not support table format {} with file format {}", diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index fb2c3578a34922..79d161902038c5 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -85,7 +85,6 @@ #include "format/orc/orc_file_reader.h" #include "format/table/iceberg_default_value.h" #include "format/table/iceberg_reader.h" -#include "format/table/transactional_hive_common.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" #include "orc/Exceptions.hh" @@ -619,7 +618,7 @@ Status OrcReader::init_schema_reader() { Status OrcReader::get_parsed_schema(std::vector* col_names, std::vector* col_types) { - const auto& root_type = _is_acid ? remove_acid(_reader->getType()) : _reader->getType(); + const auto& root_type = _reader->getType(); for (int i = 0; i < root_type.getSubtypeCount(); ++i) { col_names->emplace_back(root_type.getFieldName(i)); col_types->emplace_back(convert_to_doris_type(root_type.getSubtype(i))); @@ -653,22 +652,8 @@ Status OrcReader::_init_read_columns() { build_column_id_map(&root_type, 0); VLOG_DEBUG << "[OrcReader] Total mapped columns: " << _column_id_to_file_type.size(); - if (_is_acid) { - for (uint64_t i = 0; i < root_type.getSubtypeCount(); ++i) { - if (root_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) { - auto row_orc_type = root_type.getSubtype(i); - for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) { - _type_map.emplace(TransactionalHive::ROW + "." + row_orc_type->getFieldName(j), - row_orc_type->getSubtype(j)); - } - } else { - _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); - } - } - } else { - for (int i = 0; i < root_type.getSubtypeCount(); ++i) { - _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); - } + for (int i = 0; i < root_type.getSubtypeCount(); ++i) { + _type_map.emplace(root_type.getFieldName(i), root_type.getSubtype(i)); } return Status::OK(); @@ -688,34 +673,6 @@ void OrcReader::_init_file_column_mapping() { } } -bool OrcReader::_check_acid_schema(const orc::Type& type) { - if (orc::TypeKind::STRUCT == type.getKind()) { - if (type.getSubtypeCount() != TransactionalHive::ACID_COLUMN_NAMES.size()) { - return false; - } - for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) { - const std::string& field_name = type.getFieldName(i); - std::string field_name_lower_case = field_name; - std::transform(field_name.begin(), field_name.end(), field_name_lower_case.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (field_name_lower_case != TransactionalHive::ACID_COLUMN_NAMES_LOWER_CASE[i]) { - return false; - } - } - } else { - return false; - } - return true; -} - -const orc::Type& OrcReader::remove_acid(const orc::Type& type) { - if (_check_acid_schema(type)) { - return *(type.getSubtype(TransactionalHive::ROW_OFFSET)); - } else { - return type; - } -} - // orc only support LONG, FLOAT, STRING, DATE, DECIMAL, TIMESTAMP, BOOLEAN to push down predicates static std::unordered_map TYPEKIND_TO_PREDICATE_TYPE = { {orc::TypeKind::BYTE, orc::PredicateDataType::LONG}, @@ -1328,25 +1285,12 @@ void OrcReader::_classify_columns_for_lazy_read( const std::unordered_map>& partition_columns, const std::unordered_map& missing_columns) { - if (_is_acid) { - _lazy_read_ctx.predicate_orc_columns.insert( - _lazy_read_ctx.predicate_orc_columns.end(), - TransactionalHive::READ_ROW_COLUMN_NAMES.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES.end()); - } - for (auto& read_table_col : _read_table_cols) { _lazy_read_ctx.all_read_columns.emplace_back(read_table_col); if (!predicate_table_columns.empty()) { auto iter = predicate_table_columns.find(read_table_col); if (iter == predicate_table_columns.end()) { - if (!_is_acid || - std::find(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end(), - read_table_col) == - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()) { - _lazy_read_ctx.lazy_read_columns.emplace_back(read_table_col); - } + _lazy_read_ctx.lazy_read_columns.emplace_back(read_table_col); } else { _lazy_read_ctx.predicate_columns.first.emplace_back(iter->first); _lazy_read_ctx.predicate_columns.second.emplace_back(iter->second.second); @@ -1478,44 +1422,14 @@ Status OrcReader::_init_orc_row_reader() { const auto& selected_type = _row_reader->getSelectedType(); int idx = 0; - if (_is_acid) { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - auto sub_type = selected_type.getSubtype(i); - if (sub_type->getKind() == orc::TypeKind::STRUCT) { - for (int j = 0; j < sub_type->getSubtypeCount(); ++j) { - _colname_to_idx[TransactionalHive::ROW + "." + sub_type->getFieldName(j)] = - idx++; - } - } else { - _colname_to_idx[selected_type.getFieldName(i)] = idx++; - } - } - } else { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - _colname_to_idx[selected_type.getFieldName(i)] = idx++; - } + for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { + _colname_to_idx[selected_type.getFieldName(i)] = idx++; } _type_map.clear(); - if (_is_acid) { - for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) { - if (selected_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) { - auto row_orc_type = selected_type.getSubtype(i); - for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) { - std::string field_name = - TransactionalHive::ROW + "." + row_orc_type->getFieldName(j); - _type_map.emplace(field_name, row_orc_type->getSubtype(j)); - } - } else { - std::string field_name = selected_type.getFieldName(i); - _type_map.emplace(field_name, selected_type.getSubtype(i)); - } - } - } else { - for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { - std::string field_name = selected_type.getFieldName(i); - _type_map.emplace(field_name, selected_type.getSubtype(i)); - } + for (int i = 0; i < selected_type.getSubtypeCount(); ++i) { + std::string field_name = selected_type.getFieldName(i); + _type_map.emplace(field_name, selected_type.getSubtype(i)); } _remaining_rows = _row_reader->getNumberOfRows(); @@ -2698,7 +2612,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo #endif { SCOPED_RAW_TIMER(&_statistics.predicate_filter_time); - _build_delete_row_filter(block, _batch->numElements); std::vector columns_to_filter; int column_to_keep = block->columns(); @@ -2717,9 +2630,6 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo filter_conjuncts.emplace_back(conjunct); } std::vector filters; - if (_delete_rows_filter_ptr) { - filters.push_back(_delete_rows_filter_ptr.get()); - } IColumn::Filter result_filter(block->rows(), 1); bool can_filter_all = false; RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( @@ -2753,12 +2663,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo Block::filter_block_internal(block, columns_to_filter, result_filter)); Block::erase_useless_column(block, column_to_keep); } else { - if (_delete_rows_filter_ptr) { - _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr, start_row); - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal( - block, columns_to_filter, (*_delete_rows_filter_ptr))); - } else if (_position_delete_ordered_rowids != nullptr || - (_deletion_vector != nullptr && !_deletion_vector->isEmpty())) { + if (_position_delete_ordered_rowids != nullptr || + (_deletion_vector != nullptr && !_deletion_vector->isEmpty())) { std::unique_ptr filter(new IColumn::Filter(block->rows(), 1)); _execute_filter_position_delete_rowids(*filter, start_row); RETURN_IF_CATCH_EXCEPTION( @@ -2795,52 +2701,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo void OrcReader::_fill_batch_vec(std::vector& result, orc::ColumnVectorBatch* batch, int idx) { - if (_is_acid) { - for (auto* field : dynamic_cast(batch)->fields) { - if (dynamic_cast(field) != nullptr) { - for (auto* row_field : dynamic_cast(field)->fields) { - result.push_back(row_field); - } - } else { - result.push_back(field); - } - } - } else { - for (auto* field : dynamic_cast(batch)->fields) { - result.push_back(field); - } - } -} - -void OrcReader::_build_delete_row_filter(const Block* block, size_t rows) { - // transactional hive orc delete row - if (_delete_rows != nullptr) { - _delete_rows_filter_ptr = std::make_unique(rows, 1); - auto* __restrict _pos_delete_filter_data = _delete_rows_filter_ptr->data(); - const auto& original_transaction_column = assert_cast(*remove_nullable( - block->get_by_position((*_col_name_to_block_idx) - [TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE]) - .column)); - const auto& bucket_id_column = assert_cast(*remove_nullable( - block->get_by_position( - (*_col_name_to_block_idx)[TransactionalHive::BUCKET_LOWER_CASE]) - .column)); - const auto& row_id_column = assert_cast(*remove_nullable( - block->get_by_position( - (*_col_name_to_block_idx)[TransactionalHive::ROW_ID_LOWER_CASE]) - .column)); - for (int i = 0; i < rows; ++i) { - auto original_transaction = original_transaction_column.get_int(i); - auto bucket_id = bucket_id_column.get_int(i); - auto row_id = row_id_column.get_int(i); - - AcidRowID transactional_row_id = {.original_transaction = original_transaction, - .bucket = bucket_id, - .row_id = row_id}; - if (_delete_rows->contains(transactional_row_id)) { - _pos_delete_filter_data[i] = 0; - } - } + for (auto* field : dynamic_cast(batch)->fields) { + result.push_back(field); } } @@ -2877,11 +2739,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s std::vector table_col_names; table_col_names.insert(table_col_names.end(), _lazy_read_ctx.predicate_columns.first.begin(), _lazy_read_ctx.predicate_columns.first.end()); - if (_is_acid) { - table_col_names.insert(table_col_names.end(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()); - } for (auto& table_col_name : table_col_names) { auto& column_with_type_and_name = block->get_by_position((*_col_name_to_block_idx)[table_col_name]); @@ -2921,9 +2778,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s column_guard.mutable_column()->resize(size); } - // transactional hive orc delete row - _build_delete_row_filter(block, size); - _filter = std::make_unique(size, 1); auto* __restrict result_filter_data = _filter->data(); bool can_filter_all = false; @@ -2937,9 +2791,6 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s filter_conjuncts.emplace_back(conjunct); } std::vector filters; - if (_delete_rows_filter_ptr) { - filters.push_back(_delete_rows_filter_ptr.get()); - } RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( filter_conjuncts, &filters, block, _filter.get(), &can_filter_all)); @@ -2954,7 +2805,7 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s _lazy_read_ctx.predicate_partition_columns.size() + _lazy_read_ctx.predicate_missing_columns.size()); for (auto& col : table_col_names) { - // clean block to read predicate columns and acid columns + // Clear the columns used to evaluate predicates. columns_to_clear.emplace_back((*_col_name_to_block_idx)[col]); } for (auto& col : _lazy_read_ctx.predicate_partition_columns) { diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 842ff89d4291ad..64ea2466aded19 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -43,7 +43,6 @@ #include "format/format_common.h" #include "format/table/table_format_reader.h" #include "format/table/table_schema_change_helper.h" -#include "format/table/transactional_hive_common.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" @@ -207,8 +206,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { _deletion_vector = deletion_vector; } - void set_delete_rows(const AcidRowIDSet* delete_rows) { _delete_rows = delete_rows; } - Status filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, void* arg); Status fill_dict_filter_column_names( @@ -253,7 +250,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { } return true; } - static const orc::Type& remove_acid(const orc::Type& type); bool count_read_rows() override { return true; } @@ -270,8 +266,7 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { bool has_delete_operations() const override { return (_position_delete_ordered_rowids != nullptr && !_position_delete_ordered_rowids->empty()) || - (_deletion_vector != nullptr && !_deletion_vector->isEmpty()) || - (_delete_rows != nullptr && !_delete_rows->empty()); + (_deletion_vector != nullptr && !_deletion_vector->isEmpty()); } // RowPositionProvider implementation @@ -370,8 +365,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { Status _init_read_columns(); void _init_file_column_mapping(); - static bool _check_acid_schema(const orc::Type& type); - // ---- set_fill_columns sub-functions ---- // Collect predicate columns from conjuncts for lazy materialization. void _collect_predicate_columns_from_conjuncts( @@ -414,7 +407,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { void _fill_batch_vec(std::vector& result, orc::ColumnVectorBatch* batch, int idx); - void _build_delete_row_filter(const Block* block, size_t rows); Status _get_next_block_impl(Block* block, size_t* read_rows, bool* eof); void _init_system_properties(); void _init_file_description(); @@ -826,7 +818,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { size_t _decimal_scale_params_index = 0; protected: - bool _is_acid = false; // Protected so Iceberg subclasses can register synthesized columns // in on_before_init_reader. LazyReadContext _lazy_read_ctx; @@ -836,8 +827,6 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { private: std::unique_ptr _filter; - const AcidRowIDSet* _delete_rows = nullptr; - std::unique_ptr _delete_rows_filter_ptr; VExprContextSPtrs _not_single_slot_filter_conjuncts; const std::unordered_map* _slot_id_to_filter_conjuncts = nullptr; diff --git a/be/src/format/table/hive_reader.cpp b/be/src/format/table/hive_reader.cpp index 5d24c587d55f6b..f540c8cb0d67f4 100644 --- a/be/src/format/table/hive_reader.cpp +++ b/be/src/format/table/hive_reader.cpp @@ -98,7 +98,6 @@ Status HiveOrcReader::on_before_init_reader(ReaderInitContext* ctx) { ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); - // _is_acid is false by default, no need to set explicitly return Status::OK(); } diff --git a/be/src/format/table/transactional_hive_common.cpp b/be/src/format/table/transactional_hive_common.cpp deleted file mode 100644 index 3d68a72c8dcba2..00000000000000 --- a/be/src/format/table/transactional_hive_common.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/transactional_hive_common.h" - -#include - -namespace doris { - -const std::string TransactionalHive::OPERATION = "operation"; -const std::string TransactionalHive::ORIGINAL_TRANSACTION = "originalTransaction"; -const std::string TransactionalHive::BUCKET = "bucket"; -const std::string TransactionalHive::ROW_ID = "rowId"; -const std::string TransactionalHive::CURRENT_TRANSACTION = "currentTransaction"; -const std::string TransactionalHive::ROW = "row"; - -const std::string TransactionalHive::OPERATION_LOWER_CASE = "operation"; -const std::string TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE = "originaltransaction"; -const std::string TransactionalHive::BUCKET_LOWER_CASE = "bucket"; -const std::string TransactionalHive::ROW_ID_LOWER_CASE = "rowid"; -const std::string TransactionalHive::CURRENT_TRANSACTION_LOWER_CASE = "currenttransaction"; -const std::string TransactionalHive::ROW_LOWER_CASE = "row"; - -const int TransactionalHive::ROW_OFFSET = 5; - -const std::vector TransactionalHive::DELETE_ROW_PARAMS = { - {TransactionalHive::ORIGINAL_TRANSACTION, - TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE, PrimitiveType::TYPE_BIGINT}, - {TransactionalHive::BUCKET, TransactionalHive::BUCKET_LOWER_CASE, PrimitiveType::TYPE_INT}, - {TransactionalHive::ROW_ID, TransactionalHive::ROW_ID_LOWER_CASE, - PrimitiveType::TYPE_BIGINT}}; -const std::vector TransactionalHive::READ_PARAMS = { - {TransactionalHive::ORIGINAL_TRANSACTION, - TransactionalHive::ORIGINAL_TRANSACTION_LOWER_CASE, PrimitiveType::TYPE_BIGINT}, - {TransactionalHive::BUCKET, TransactionalHive::BUCKET_LOWER_CASE, PrimitiveType::TYPE_INT}, - {TransactionalHive::ROW_ID, TransactionalHive::ROW_ID_LOWER_CASE, - PrimitiveType::TYPE_BIGINT}}; -const std::vector TransactionalHive::DELETE_ROW_COLUMN_NAMES = { - DELETE_ROW_PARAMS[0].column_name, DELETE_ROW_PARAMS[1].column_name, - DELETE_ROW_PARAMS[2].column_name}; - -const std::vector TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE = { - DELETE_ROW_PARAMS[0].column_lower_case, DELETE_ROW_PARAMS[1].column_lower_case, - DELETE_ROW_PARAMS[2].column_lower_case}; - -const std::vector TransactionalHive::READ_ROW_COLUMN_NAMES = { - READ_PARAMS[0].column_name, READ_PARAMS[1].column_name, READ_PARAMS[2].column_name}; - -const std::vector TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE = { - READ_PARAMS[0].column_lower_case, READ_PARAMS[1].column_lower_case, - READ_PARAMS[2].column_lower_case}; - -const std::vector TransactionalHive::ACID_COLUMN_NAMES = { - OPERATION, ORIGINAL_TRANSACTION, BUCKET, ROW_ID, CURRENT_TRANSACTION, ROW}; - -const std::vector TransactionalHive::ACID_COLUMN_NAMES_LOWER_CASE = { - OPERATION_LOWER_CASE, ORIGINAL_TRANSACTION_LOWER_CASE, BUCKET_LOWER_CASE, - ROW_ID_LOWER_CASE, CURRENT_TRANSACTION_LOWER_CASE, ROW_LOWER_CASE}; - -const std::unordered_map TransactionalHive::DELETE_COL_NAME_TO_BLOCK_IDX = { - {DELETE_ROW_PARAMS[0].column_lower_case, 0}, - {DELETE_ROW_PARAMS[1].column_lower_case, 1}, - {DELETE_ROW_PARAMS[2].column_lower_case, 2}}; - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_common.h b/be/src/format/table/transactional_hive_common.h deleted file mode 100644 index 435294220fad24..00000000000000 --- a/be/src/format/table/transactional_hive_common.h +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "core/data_type/define_primitive_type.h" -#include "exec/common/hash_table/phmap_fwd_decl.h" - -namespace doris { -struct TransactionalHive { - static const std::string OPERATION; - static const std::string ORIGINAL_TRANSACTION; - static const std::string BUCKET; - static const std::string ROW_ID; - static const std::string CURRENT_TRANSACTION; - static const std::string ROW; - static const std::string OPERATION_LOWER_CASE; - static const std::string ORIGINAL_TRANSACTION_LOWER_CASE; - static const std::string BUCKET_LOWER_CASE; - static const std::string ROW_ID_LOWER_CASE; - static const std::string CURRENT_TRANSACTION_LOWER_CASE; - static const std::string ROW_LOWER_CASE; - static const int ROW_OFFSET; - struct Param { - const std::string column_name; - const std::string column_lower_case; - const PrimitiveType type; - }; - static const std::vector DELETE_ROW_PARAMS; - static const std::vector READ_PARAMS; - static const std::vector DELETE_ROW_COLUMN_NAMES; - static const std::vector DELETE_ROW_COLUMN_NAMES_LOWER_CASE; - static const std::vector READ_ROW_COLUMN_NAMES; - static const std::vector READ_ROW_COLUMN_NAMES_LOWER_CASE; - static const std::vector ACID_COLUMN_NAMES; - static const std::vector ACID_COLUMN_NAMES_LOWER_CASE; - - static const std::unordered_map DELETE_COL_NAME_TO_BLOCK_IDX; -}; - -// ACID row identifier for transactional Hive tables, used for delete row matching. -// Placed here (not in TransactionalHiveReader) to avoid circular dependency with OrcReader. -struct AcidRowID { - int64_t original_transaction; - int64_t bucket; - int64_t row_id; - - struct Hash { - size_t operator()(const AcidRowID& transactional_row_id) const { - size_t hash_value = 0; - hash_value ^= std::hash {}(transactional_row_id.original_transaction) + - 0x9e3779b9 + (hash_value << 6) + (hash_value >> 2); - hash_value ^= std::hash {}(transactional_row_id.bucket) + 0x9e3779b9 + - (hash_value << 6) + (hash_value >> 2); - hash_value ^= std::hash {}(transactional_row_id.row_id) + 0x9e3779b9 + - (hash_value << 6) + (hash_value >> 2); - return hash_value; - } - }; - - struct Eq { - bool operator()(const AcidRowID& lhs, const AcidRowID& rhs) const { - return lhs.original_transaction == rhs.original_transaction && - lhs.bucket == rhs.bucket && lhs.row_id == rhs.row_id; - } - }; -}; - -using AcidRowIDSet = flat_hash_set; - -inline bool operator<(const AcidRowID& lhs, const AcidRowID& rhs) { - if (lhs.original_transaction != rhs.original_transaction) { - return lhs.original_transaction < rhs.original_transaction; - } else if (lhs.bucket != rhs.bucket) { - return lhs.bucket < rhs.bucket; - } else if (lhs.row_id != rhs.row_id) { - return lhs.row_id < rhs.row_id; - } else { - return false; - } -} - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_reader.cpp b/be/src/format/table/transactional_hive_reader.cpp deleted file mode 100644 index cef77926f25445..00000000000000 --- a/be/src/format/table/transactional_hive_reader.cpp +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/transactional_hive_reader.h" - -#include - -#include - -#include "core/data_type/data_type_factory.hpp" -#include "format/orc/vorc_reader.h" -#include "format/table/table_schema_change_helper.h" -#include "transactional_hive_common.h" - -namespace doris { - -namespace io { -struct IOContext; -} // namespace io -class VExprContext; -} // namespace doris - -namespace doris { - -TransactionalHiveReader::TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, - const TFileRangeDesc& range, size_t batch_size, - const std::string& ctz, io::IOContext* io_ctx, - FileMetaCache* meta_cache) - : OrcReader(profile, state, params, range, batch_size, ctz, io_ctx, meta_cache, false) { - _init_transactional_hive_profile(); -} - -TransactionalHiveReader::TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, - const TFileRangeDesc& range, size_t batch_size, - const std::string& ctz, - std::shared_ptr io_ctx_holder, - FileMetaCache* meta_cache) - : OrcReader(profile, state, params, range, batch_size, ctz, std::move(io_ctx_holder), - meta_cache, false) { - _init_transactional_hive_profile(); -} - -void TransactionalHiveReader::_init_transactional_hive_profile() { - static const char* transactional_hive_profile = "TransactionalHiveProfile"; - ADD_TIMER(get_profile(), transactional_hive_profile); - _transactional_orc_profile.num_delete_files = ADD_CHILD_COUNTER( - get_profile(), "NumDeleteFiles", TUnit::UNIT, transactional_hive_profile); - _transactional_orc_profile.num_delete_rows = ADD_CHILD_COUNTER( - get_profile(), "NumDeleteRows", TUnit::UNIT, transactional_hive_profile); - _transactional_orc_profile.delete_files_read_time = - ADD_CHILD_TIMER(get_profile(), "DeleteFileReadTime", transactional_hive_profile); -} - -// ============================================================================ -// on_before_init_reader: ACID schema mapping -// ============================================================================ -Status TransactionalHiveReader::on_before_init_reader(ReaderInitContext* ctx) { - _column_descs = ctx->column_descs; - _fill_col_name_to_block_idx = ctx->col_name_to_block_idx; - RETURN_IF_ERROR(_extract_partition_values(*ctx->range, ctx->tuple_descriptor, - _fill_partition_values, - &_fill_partition_value_is_null)); - for (const auto& desc : *ctx->column_descs) { - if (desc.category == ColumnCategory::REGULAR || - desc.category == ColumnCategory::GENERATED) { - _col_names.push_back(desc.name); - } else if (desc.category == ColumnCategory::SYNTHESIZED && - desc.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { - auto topn_row_id_column_iter = _create_topn_row_id_column_iterator(); - this->register_synthesized_column_handler( - desc.name, - [iter = std::move(topn_row_id_column_iter), this, &desc]( - Block* block, size_t rows) -> Status { - return fill_topn_row_id(iter, desc.name, block, rows); - }); - continue; - } - } - - _is_acid = true; - // Add ACID column names (originalTransaction, bucket, rowId, etc.) - _col_names.insert(_col_names.end(), TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()); - ctx->column_names = _col_names; - - // Get ORC file type - const orc::Type* orc_type_ptr = nullptr; - RETURN_IF_ERROR(get_file_type(&orc_type_ptr)); - const auto& orc_type = *orc_type_ptr; - - // Add ACID metadata columns to table_info_node - for (auto idx = 0; idx < TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.size(); idx++) { - table_info_node_ptr->add_children(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE[idx], - TransactionalHive::READ_ROW_COLUMN_NAMES[idx], - std::make_shared()); - } - - // https://issues.apache.org/jira/browse/HIVE-15190 - auto row_orc_type = orc_type.getSubtype(TransactionalHive::ROW_OFFSET); - std::vector row_names; - std::map row_names_map; - for (uint64_t idx = 0; idx < row_orc_type->getSubtypeCount(); idx++) { - const auto& file_column_name = row_orc_type->getFieldName(idx); - row_names.emplace_back(file_column_name); - row_names_map.emplace(file_column_name, idx); - } - - // Match table columns to file columns by name - for (const auto& slot : ctx->tuple_descriptor->slots()) { - const auto& slot_name = slot->col_name(); - - if (std::count(TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end(), slot_name) > 0) { - return Status::InternalError("Column {} conflicts with ACID metadata column", - slot_name); - } - - if (row_names_map.contains(slot_name)) { - std::shared_ptr child_node = nullptr; - RETURN_IF_ERROR(BuildTableInfoUtil::by_orc_name( - slot->type(), row_orc_type->getSubtype(row_names_map[slot_name]), child_node)); - auto file_column_name = fmt::format( - "{}.{}", TransactionalHive::ACID_COLUMN_NAMES[TransactionalHive::ROW_OFFSET], - slot_name); - table_info_node_ptr->add_children(slot_name, file_column_name, child_node); - } else { - table_info_node_ptr->add_not_exist_children(slot_name); - } - } - ctx->table_info_node = table_info_node_ptr; - return Status::OK(); -} - -// ============================================================================ -// on_after_init_reader: read delete delta files -// ============================================================================ -Status TransactionalHiveReader::on_after_init_reader(ReaderInitContext* /*ctx*/) { - std::string data_file_path = get_scan_range().path; - // the path in _range has the namenode prefix removed, - // and the file_path in delete file is full path, so we should add it back. - if (get_scan_params().__isset.hdfs_params && get_scan_params().hdfs_params.__isset.fs_name) { - std::string fs_name = get_scan_params().hdfs_params.fs_name; - if (!starts_with(data_file_path, fs_name)) { - data_file_path = fs_name + data_file_path; - } - } - - std::vector delete_file_col_names; - int64_t num_delete_rows = 0; - int64_t num_delete_files = 0; - std::filesystem::path file_path(data_file_path); - - // bucket_xxx_attemptId => bucket_xxx - auto remove_bucket_attemptId = [](const std::string& str) { - re2::RE2 pattern("^bucket_\\d+_\\d+$"); - if (re2::RE2::FullMatch(str, pattern)) { - size_t pos = str.rfind('_'); - if (pos != std::string::npos) { - return str.substr(0, pos); - } - } - return str; - }; - - SCOPED_TIMER(_transactional_orc_profile.delete_files_read_time); - for (const auto& delete_delta : - get_scan_range().table_format_params.transactional_hive_params.delete_deltas) { - const std::string file_name = file_path.filename().string(); - - std::vector delete_delta_file_names; - for (const auto& x : delete_delta.file_names) { - delete_delta_file_names.emplace_back(remove_bucket_attemptId(x)); - } - auto iter = std::find(delete_delta_file_names.begin(), delete_delta_file_names.end(), - remove_bucket_attemptId(file_name)); - if (iter == delete_delta_file_names.end()) { - continue; - } - auto delete_file = - fmt::format("{}/{}", delete_delta.directory_location, - delete_delta.file_names[iter - delete_delta_file_names.begin()]); - - TFileRangeDesc delete_range; - delete_range.__set_fs_name(get_scan_range().fs_name); - delete_range.path = delete_file; - delete_range.start_offset = 0; - delete_range.size = -1; - delete_range.file_size = -1; - - OrcReader delete_reader(get_profile(), get_state(), get_scan_params(), delete_range, - 256 /*batch_size*/, get_state()->timezone(), get_io_ctx(), - _meta_cache, false); - - auto acid_info_node = std::make_shared(); - for (auto idx = 0; idx < TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.size(); - idx++) { - auto const& table_column_name = - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE[idx]; - auto const& file_column_name = TransactionalHive::DELETE_ROW_COLUMN_NAMES[idx]; - acid_info_node->add_children(table_column_name, file_column_name, - std::make_shared()); - } - - OrcInitContext delete_ctx; - delete_ctx.column_names.assign( - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.begin(), - TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.end()); - delete_ctx.col_name_to_block_idx = const_cast*>( - &TransactionalHive::DELETE_COL_NAME_TO_BLOCK_IDX); - delete_ctx.table_info_node = acid_info_node; - RETURN_IF_ERROR(delete_reader.init_reader(&delete_ctx)); - - bool eof = false; - while (!eof) { - Block block; - for (const auto& i : TransactionalHive::DELETE_ROW_PARAMS) { - DataTypePtr data_type = DataTypeFactory::instance().create_data_type(i.type, false); - MutableColumnPtr data_column = data_type->create_column(); - block.insert(ColumnWithTypeAndName(std::move(data_column), data_type, - i.column_lower_case)); - } - eof = false; - size_t read_rows = 0; - RETURN_IF_ERROR(delete_reader.get_next_block(&block, &read_rows, &eof)); - if (read_rows > 0) { - static int ORIGINAL_TRANSACTION_INDEX = 0; - static int BUCKET_ID_INDEX = 1; - static int ROW_ID_INDEX = 2; - const auto& original_transaction_column = assert_cast( - *block.get_by_position(ORIGINAL_TRANSACTION_INDEX).column); - const auto& bucket_id_column = assert_cast( - *block.get_by_position(BUCKET_ID_INDEX).column); - const auto& row_id_column = assert_cast( - *block.get_by_position(ROW_ID_INDEX).column); - - DCHECK_EQ(original_transaction_column.size(), read_rows); - DCHECK_EQ(bucket_id_column.size(), read_rows); - DCHECK_EQ(row_id_column.size(), read_rows); - - for (int i = 0; i < read_rows; ++i) { - Int64 original_transaction = original_transaction_column.get_int(i); - Int64 bucket_id = bucket_id_column.get_int(i); - Int64 row_id = row_id_column.get_int(i); - AcidRowID delete_row_id = {original_transaction, bucket_id, row_id}; - _acid_delete_rows.insert(delete_row_id); - ++num_delete_rows; - } - } - } - ++num_delete_files; - } - if (num_delete_rows > 0) { - set_push_down_agg_type(TPushAggOp::NONE); - set_delete_rows(&_acid_delete_rows); - COUNTER_UPDATE(_transactional_orc_profile.num_delete_files, num_delete_files); - COUNTER_UPDATE(_transactional_orc_profile.num_delete_rows, num_delete_rows); - } - return Status::OK(); -} - -// ============================================================================ -// on_before_read_block: expand ACID columns into block -// TODO: Consider caching ACID column templates at init time to avoid repeated -// create_column + map update on every batch. Requires a block template mechanism. -// ============================================================================ -Status TransactionalHiveReader::on_before_read_block(Block* block) { - for (const auto& i : TransactionalHive::READ_PARAMS) { - DataTypePtr data_type = DataTypeFactory::instance().create_data_type(i.type, false); - MutableColumnPtr data_column = data_type->create_column(); - (*col_name_to_block_idx_ref())[i.column_lower_case] = block->columns(); - block->insert( - ColumnWithTypeAndName(std::move(data_column), data_type, i.column_lower_case)); - } - return Status::OK(); -} - -// ============================================================================ -// on_after_read_block: shrink ACID columns from block -// ============================================================================ -Status TransactionalHiveReader::on_after_read_block(Block* block, size_t* /*read_rows*/) { - Block::erase_useless_column(block, block->columns() - TransactionalHive::READ_PARAMS.size()); - for (const auto& i : TransactionalHive::READ_PARAMS) { - col_name_to_block_idx_ref()->erase(i.column_lower_case); - } - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/format/table/transactional_hive_reader.h b/be/src/format/table/transactional_hive_reader.h deleted file mode 100644 index 1f8c26a7ff298f..00000000000000 --- a/be/src/format/table/transactional_hive_reader.h +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/factory_creator.h" -#include "common/status.h" -#include "format/orc/vorc_reader.h" -#include "format/table/table_schema_change_helper.h" -#include "format/table/transactional_hive_common.h" - -namespace doris { -class RuntimeState; -class TFileRangeDesc; -class TFileScanRangeParams; - -namespace io { -struct IOContext; -} // namespace io - -class Block; -class ShardedKVCache; -class VExprContext; - -// TransactionalHiveReader: directly inherits OrcReader (no composition wrapping). -// ACID column expansion/shrinking done via on_before_read_block/on_after_read_block hooks. -// Delete delta reading done via on_after_init_reader hook. -class TransactionalHiveReader final : public OrcReader, public TableSchemaChangeHelper { - ENABLE_FACTORY_CREATOR(TransactionalHiveReader); - -public: - TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, const TFileRangeDesc& range, - size_t batch_size, const std::string& ctz, io::IOContext* io_ctx, - FileMetaCache* meta_cache = nullptr); - TransactionalHiveReader(RuntimeProfile* profile, RuntimeState* state, - const TFileScanRangeParams& params, const TFileRangeDesc& range, - size_t batch_size, const std::string& ctz, - std::shared_ptr io_ctx_holder, - FileMetaCache* meta_cache = nullptr); - ~TransactionalHiveReader() final = default; - -protected: - // Hook: ACID schema mapping (add transactional columns, map row.* fields) - Status on_before_init_reader(ReaderInitContext* ctx) override; - - // Hook: read delete delta files - Status on_after_init_reader(ReaderInitContext* /*ctx*/) override; - - // Hook: expand ACID columns into block before reading - Status on_before_read_block(Block* block) override; - - // Hook: shrink ACID columns from block after reading - Status on_after_read_block(Block* block, size_t* read_rows) override; - -private: - void _init_transactional_hive_profile(); - - struct TransactionalHiveProfile { - RuntimeProfile::Counter* num_delete_files = nullptr; - RuntimeProfile::Counter* num_delete_rows = nullptr; - RuntimeProfile::Counter* delete_files_read_time = nullptr; - }; - - TransactionalHiveProfile _transactional_orc_profile; - AcidRowIDSet _acid_delete_rows; - std::vector _col_names; -}; - -} // namespace doris diff --git a/be/test/exec/scan/file_scan_range_utils_test.cpp b/be/test/exec/scan/file_scan_range_utils_test.cpp index bc8bbfaa465535..46e1cc7898a014 100644 --- a/be/test/exec/scan/file_scan_range_utils_test.cpp +++ b/be/test/exec/scan/file_scan_range_utils_test.cpp @@ -21,6 +21,31 @@ namespace doris { +TEST(FileScanRangeUtilsTest, RemovedFormatsCannotUseLegacyOrMetadataReads) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_ORC); + EXPECT_TRUE(validate_file_scan_support(params, &range).ok()); + for (const auto* format : {"transactional_hive", "lakesoul"}) { + TTableFormatFileDesc table; + table.__set_table_format_type(format); + table.__set_table_level_row_count(10); + range.__set_table_format_params(table); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); + range.__isset.table_format_params = false; + params.__set_table_format_params(table); + EXPECT_FALSE(validate_file_scan_support(params).ok()); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); + params.__isset.table_format_params = false; + } + params.__set_format_type(TFileFormatType::FORMAT_AVRO); + EXPECT_FALSE(validate_file_scan_support(params).ok()); + params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + range.__set_format_type(TFileFormatType::FORMAT_AVRO); + EXPECT_FALSE(validate_file_scan_support(params, &range).ok()); +} + TEST(FileScanRangeUtilsTest, IcebergFilesCannotBeIgnored) { TFileRangeDesc range; range.table_format_params.__set_table_format_type("iceberg"); diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index c6c5c68051095d..3c13bb1a8342eb 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -495,11 +495,8 @@ TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesAndLoads) params.__set_format_type(TFileFormatType::FORMAT_ORC); TTableFormatFileDesc table_format_params; - table_format_params.__set_table_format_type("transactional_hive"); + table_format_params.__set_table_format_type("hive"); params.__set_table_format_params(table_format_params); - EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - - params.table_format_params.__set_table_format_type("hive"); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); query_options.__set_enable_file_scanner_v2(false); @@ -803,7 +800,7 @@ TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) { const auto unsupported = range_with_format("lakesoul", TFileFormatType::FORMAT_PARQUET); const auto status = FileScannerV2::TEST_validate_scan_range(params, unsupported); EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("lakesoul"), std::string::npos); + EXPECT_NE(status.to_string().find("LakeSoul"), std::string::npos); } // Scenario: external file scan IO contexts are query readers for SELECT so the shared file-cache diff --git a/be/test/format/condition_cache_test.cpp b/be/test/format/condition_cache_test.cpp index c0e9c86af64853..e8366fcc3dc776 100644 --- a/be/test/format/condition_cache_test.cpp +++ b/be/test/format/condition_cache_test.cpp @@ -27,7 +27,6 @@ #include "format/generic_reader.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" -#include "format/table/transactional_hive_common.h" namespace doris::vectorized { @@ -506,24 +505,6 @@ TEST_F(ConditionCacheDeleteOpsTest, OrcWithPositionDeletes_CacheSkipped) { EXPECT_EQ(cache, nullptr); } -// -- OrcReader: with ACID deletes -> cache skipped -- -TEST_F(ConditionCacheDeleteOpsTest, OrcWithAcidDeletes_CacheSkipped) { - TFileScanRangeParams params; - TFileRangeDesc range; - auto reader = OrcReader::create_unique(params, range, 4064, "", nullptr); - AcidRowIDSet acid_deletes; - acid_deletes.insert({1, 0, 5}); - reader->set_delete_rows(&acid_deletes); - - bool hit = false; - std::shared_ptr> cache; - std::shared_ptr ctx; - simulate_init_condition_cache(reader.get(), 99, "/data/file.orc", hit, cache, ctx); - - EXPECT_EQ(ctx, nullptr); - EXPECT_EQ(cache, nullptr); -} - // -- MockReader: with deletes (simulating Iceberg/Hive with inner deletes) -> cache skipped -- // In the new architecture, Iceberg readers inherit ParquetReader/OrcReader directly (CRTP), // so has_delete_operations() is resolved through the base reader. We use MockFileFormatReader diff --git a/be/test/format/orc/orc_reader_init_column_test.cpp b/be/test/format/orc/orc_reader_init_column_test.cpp index 9c685176196262..553d3046a60bef 100644 --- a/be/test/format/orc/orc_reader_init_column_test.cpp +++ b/be/test/format/orc/orc_reader_init_column_test.cpp @@ -68,164 +68,4 @@ TEST_F(OrcReaderInitColumnTest, InitReadColumn) { } } -TEST_F(OrcReaderInitColumnTest, CheckAcidSchemaTest) { - using namespace orc; - TFileScanRangeParams params; - TFileRangeDesc range; - auto _reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - // 1. Test standard ACID schema - { - // Create standard ACID structure - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("row", createStructType()); - - ASSERT_TRUE(_reader->_check_acid_schema(*acid_type)); - } - - // 2. Test case-insensitive field names - { - auto acid_type = createStructType(); - acid_type->addStructField("OPERATION", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("OriginalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("Bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("ROWID", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTRANSACTION", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("ROW", createStructType()); - - ASSERT_TRUE(_reader->_check_acid_schema(*acid_type)); - } - - // 3. Test non-ACID schema - field count mismatch - { - auto non_acid_type = createStructType(); - non_acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - non_acid_type->addStructField("originalTransaction", - createPrimitiveType(orc::TypeKind::LONG)); - // Only added two fields - - ASSERT_FALSE(_reader->_check_acid_schema(*non_acid_type)); - } - - // 4. Test non-ACID schema - field name mismatch - { - auto wrong_name_type = createStructType(); - wrong_name_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - wrong_name_type->addStructField("wrongName", createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - wrong_name_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("currentTransaction", - createPrimitiveType(orc::TypeKind::LONG)); - wrong_name_type->addStructField("row", createStructType()); - - ASSERT_FALSE(_reader->_check_acid_schema(*wrong_name_type)); - } - - // 5. Test non-struct type - { - auto int_type = createPrimitiveType(orc::TypeKind::INT); - ASSERT_FALSE(_reader->_check_acid_schema(*int_type)); - - auto string_type = createPrimitiveType(orc::TypeKind::STRING); - ASSERT_FALSE(_reader->_check_acid_schema(*string_type)); - } -} - -TEST_F(OrcReaderInitColumnTest, RemoveAcidTest) { - using namespace orc; - TFileScanRangeParams params; - TFileRangeDesc range; - auto _reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - // 1. Test removing ACID info from ACID schema - { - // Create ACID schema - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - - // Create actual data structure - auto row_type = createStructType(); - row_type->addStructField("id", createPrimitiveType(orc::TypeKind::INT)); - row_type->addStructField("name", createPrimitiveType(orc::TypeKind::STRING)); - acid_type->addStructField("row", std::move(row_type)); - - // Verify that after removing ACID we get the type of the row field - const orc::Type& removed_type = _reader->remove_acid(*acid_type); - ASSERT_EQ(removed_type.getKind(), orc::TypeKind::STRUCT); - ASSERT_EQ(removed_type.getSubtypeCount(), 2); // id and name fields - ASSERT_EQ(removed_type.getFieldName(0), "id"); - ASSERT_EQ(removed_type.getFieldName(1), "name"); - } - - // 2. Test that non-ACID schema remains unchanged - { - // Create normal schema - auto normal_type = createStructType(); - normal_type->addStructField("field1", createPrimitiveType(orc::TypeKind::INT)); - normal_type->addStructField("field2", createPrimitiveType(orc::TypeKind::STRING)); - - const orc::Type& result_type = _reader->remove_acid(*normal_type); - ASSERT_EQ(&result_type, normal_type.get()); // Should return the same type - ASSERT_EQ(result_type.getSubtypeCount(), 2); - ASSERT_EQ(result_type.getFieldName(0), "field1"); - ASSERT_EQ(result_type.getFieldName(1), "field2"); - } - - // 3. Test primitive types (non-struct) remain unchanged - { - auto int_type = createPrimitiveType(orc::TypeKind::INT); - const orc::Type& result_type = _reader->remove_acid(*int_type); - ASSERT_EQ(&result_type, int_type.get()); - ASSERT_EQ(result_type.getKind(), orc::TypeKind::INT); - } - - // 4. Test complex nested ACID schema - { - // Create nested ACID schema - auto acid_type = createStructType(); - acid_type->addStructField("operation", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("originalTransaction", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("bucket", createPrimitiveType(orc::TypeKind::INT)); - acid_type->addStructField("rowId", createPrimitiveType(orc::TypeKind::LONG)); - acid_type->addStructField("currentTransaction", createPrimitiveType(orc::TypeKind::LONG)); - - // Create complex row structure - auto row_type = createStructType(); - - // Add basic fields - row_type->addStructField("id", createPrimitiveType(orc::TypeKind::INT)); - - // Add array field - auto array_type = createListType(createPrimitiveType(orc::TypeKind::STRING)); - row_type->addStructField("tags", std::move(array_type)); - - // Add Map field - auto map_type = createMapType(createPrimitiveType(orc::TypeKind::STRING), - createPrimitiveType(orc::TypeKind::INT)); - row_type->addStructField("properties", std::move(map_type)); - - acid_type->addStructField("row", std::move(row_type)); - - // Verify structure after removing ACID - const orc::Type& removed_type = _reader->remove_acid(*acid_type); - ASSERT_EQ(removed_type.getKind(), orc::TypeKind::STRUCT); - ASSERT_EQ(removed_type.getSubtypeCount(), 3); // id, tags, properties - ASSERT_EQ(removed_type.getFieldName(0), "id"); - ASSERT_EQ(removed_type.getFieldName(1), "tags"); - ASSERT_EQ(removed_type.getFieldName(2), "properties"); - - // Verify field types - ASSERT_EQ(removed_type.getSubtype(0)->getKind(), orc::TypeKind::INT); - ASSERT_EQ(removed_type.getSubtype(1)->getKind(), orc::TypeKind::LIST); - ASSERT_EQ(removed_type.getSubtype(2)->getKind(), orc::TypeKind::MAP); - } -} - } // namespace doris diff --git a/build-for-release.sh b/build-for-release.sh index 5282a723e2938c..336ba30e95c88a 100755 --- a/build-for-release.sh +++ b/build-for-release.sh @@ -147,7 +147,7 @@ echo "CLOUD: ${OUTPUT_CLOUD}" sh build.sh --clean && USE_AVX2="${_USE_AVX2}" sh build.sh && - USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool --be-extension-ignore avro-scanner + USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool echo "Begin to pack" rm -rf "${OUTPUT}" diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql deleted file mode 100755 index 79b871822d1ea5..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run38.hql +++ /dev/null @@ -1,69 +0,0 @@ -drop table if exists `avro_all_types`; -create table `avro_all_types`( -`t_null_string` string, -`t_null_varchar` varchar(65535), -`t_null_char` char(10), -`t_null_array_int` array, -`t_null_decimal_precision_2` decimal(2,1), -`t_null_decimal_precision_4` decimal(4,2), -`t_null_decimal_precision_8` decimal(8,4), -`t_null_decimal_precision_17` decimal(17,8), -`t_null_decimal_precision_18` decimal(18,8), -`t_null_decimal_precision_38` decimal(38,16), -`t_empty_string` string, -`t_string` string, -`t_empty_varchar` varchar(65535), -`t_varchar` varchar(65535), -`t_varchar_max_length` varchar(65535), -`t_char` char(10), -`t_int` int, -`t_bigint` bigint, -`t_float` float, -`t_double` double, -`t_boolean_true` boolean, -`t_boolean_false` boolean, -`t_date` date, -`t_timestamp` timestamp, -`t_decimal_precision_2` decimal(2,1), -`t_decimal_precision_4` decimal(4,2), -`t_decimal_precision_8` decimal(8,4), -`t_decimal_precision_17` decimal(17,8), -`t_decimal_precision_18` decimal(18,8), -`t_decimal_precision_38` decimal(38,16), -`t_binary` binary, -`t_map_string` map, -`t_array_empty` array, -`t_array_string` array, -`t_array_int` array, -`t_array_bigint` array, -`t_array_float` array, -`t_array_double` array, -`t_array_boolean` array, -`t_array_varchar` array, -`t_array_char` array, -`t_array_date` array, -`t_array_timestamp` array, -`t_array_decimal_precision_2` array, -`t_array_decimal_precision_4` array, -`t_array_decimal_precision_8` array, -`t_array_decimal_precision_17` array, -`t_array_decimal_precision_18` array, -`t_array_decimal_precision_38` array, -`t_struct_bigint` struct, -`t_complex` map>>, -`t_struct_nested` struct>, -`t_struct_null` struct, -`t_struct_non_nulls_after_nulls` struct, -`t_array_string_starting_with_nulls` array, -`t_array_string_with_nulls_in_between` array, -`t_array_string_ending_with_nulls` array, -`t_array_string_all_nulls` array -) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' -LOCATION - '/user/doris/preinstalled_data/avro/avro_all_types'; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro deleted file mode 100644 index c66017bb6243ee..00000000000000 Binary files a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/all_type.avro and /dev/null differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types deleted file mode 100644 index 169ad69cb43a67..00000000000000 Binary files a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/avro/avro_all_types/avro_all_types and /dev/null differ diff --git a/docker/thirdparties/docker-compose/lakesoul/lakesoul.yaml.tpl b/docker/thirdparties/docker-compose/lakesoul/lakesoul.yaml.tpl deleted file mode 100644 index 3d5346a6118bbd..00000000000000 --- a/docker/thirdparties/docker-compose/lakesoul/lakesoul.yaml.tpl +++ /dev/null @@ -1,64 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3' - -services: - lakesoul-meta-db: - image: postgres:14.5 - container_name: lakesoul-test-pg - hostname: lakesoul-docker-compose-env-lakesoul-meta-db-1 - ports: - - "5432:5432" - restart: always - environment: - POSTGRES_PASSWORD: lakesoul_test - POSTGRES_USER: lakesoul_test - POSTGRES_DB: lakesoul_test - command: - - "postgres" - - "-c" - - "max_connections=4096" - - "-c" - - "default_transaction_isolation=serializable" - volumes: - - ./meta_init.sql:/docker-entrypoint-initdb.d/meta_init.sql - - ./meta_cleanup.sql:/meta_cleanup.sql - -# minio: -# image: bitnami/minio:latest -# ports: -# - "9000:9000" -# - "9001:9001" -# environment: -# MINIO_DEFAULT_BUCKETS: lakesoul-test-bucket:public -# MINIO_ROOT_USER: minioadmin1 -# MINIO_ROOT_PASSWORD: minioadmin1 -# healthcheck: -# test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] -# interval: 3s -# timeout: 5s -# retries: 3 -# hostname: minio -# profiles: ["s3"] - - -networks: - default: - driver: bridge - ipam: - driver: default diff --git a/docker/thirdparties/docker-compose/lakesoul/meta_cleanup.sql b/docker/thirdparties/docker-compose/lakesoul/meta_cleanup.sql deleted file mode 100644 index dd25a7ee958879..00000000000000 --- a/docker/thirdparties/docker-compose/lakesoul/meta_cleanup.sql +++ /dev/null @@ -1,11 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -delete from namespace; -insert into namespace(namespace, properties, comment) values ('default', '{}', ''); -delete from data_commit_info; -delete from table_info; -delete from table_path_id; -delete from table_name_id; -delete from partition_info; diff --git a/docker/thirdparties/docker-compose/lakesoul/meta_init.sql b/docker/thirdparties/docker-compose/lakesoul/meta_init.sql deleted file mode 100644 index 53bb64067140c3..00000000000000 --- a/docker/thirdparties/docker-compose/lakesoul/meta_init.sql +++ /dev/null @@ -1,144 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -create table if not exists namespace -( - namespace text, - properties json, - comment text, - domain text default 'public', - primary key (namespace) -); - -insert into namespace(namespace, properties, comment) values ('default', '{}', '') -ON CONFLICT DO NOTHING; - -create table if not exists table_info -( - table_id text, - table_namespace text default 'default', - table_name text, - table_path text, - table_schema text, - properties json, - partitions text, - domain text default 'public', - primary key (table_id) -); - -create table if not exists table_name_id -( - table_name text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_name, table_namespace) -); - -create table if not exists table_path_id -( - table_path text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_path) -); - -DO -$$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'data_file_op') THEN - create type data_file_op as - ( - path text, - file_op text, - size bigint, - file_exist_cols text - ); - END IF; - END -$$; - -create table if not exists data_commit_info -( - table_id text, - partition_desc text, - commit_id UUID, - file_ops data_file_op[], - commit_op text, - committed boolean default 'false', - timestamp bigint, - domain text default 'public', - primary key (table_id, partition_desc, commit_id) -); - -create table if not exists partition_info -( - table_id text, - partition_desc text, - version int, - commit_op text, - timestamp bigint DEFAULT (date_part('epoch'::text, now()) * (1000)::double precision), - snapshot UUID[], - expression text, - domain text default 'public', - primary key (table_id, partition_desc, version) -); - -CREATE OR REPLACE FUNCTION partition_insert() RETURNS TRIGGER AS -$$ -DECLARE - rs_version integer; - rs_table_path text; - rs_table_namespace text; -BEGIN - if NEW.commit_op <> 'CompactionCommit' then - select version - INTO rs_version - from partition_info - where table_id = NEW.table_id - and partition_desc = NEW.partition_desc - and version != NEW.version - and commit_op = 'CompactionCommit' - order by version desc - limit 1; - if rs_version >= 0 then - if NEW.version - rs_version >= 10 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - else - if NEW.version >= 10 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - end if; - RETURN NULL; - end if; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER partition_table_change - AFTER INSERT - ON partition_info - FOR EACH ROW -EXECUTE PROCEDURE partition_insert(); - -create table if not exists global_config -( - key text, - value text, - primary key (key) -); diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 0f83913380b2de..1c5ec1d5f83414 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -76,12 +76,12 @@ Usage: $0 --hive-modules comma separated hive modules to refresh All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio,fluss + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,kerberos,ranger,polaris,minio,fluss " exit 1 } DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,hudi,db2,oceanbase,kerberos,minio" -ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris,fluss" +ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,ranger,polaris,fluss" COMPONENTS=$2 HELP=0 STOP=0 @@ -294,8 +294,6 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_DB2=1 elif [[ "${element}"x == "oceanbase"x ]];then RUN_OCEANBASE=1 - elif [[ "${element}"x == "lakesoul"x ]]; then - RUN_LAKESOUL=1 elif [[ "${element}"x == "kerberos"x ]]; then RUN_KERBEROS=1 elif [[ "${element}"x == "minio"x ]]; then @@ -1542,43 +1540,6 @@ start_fluss() { compose_up_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" -d --wait } -start_lakesoul() { - echo "RUN_LAKESOUL" - cp "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml.tpl "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml - register_stack_metadata "lakesoul" "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" - compose_cmd "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" down --remove-orphans - sudo rm -rf "${ROOT}"/docker-compose/lakesoul/data - if [[ "${STOP}" -ne 1 ]]; then - echo "PREPARE_LAKESOUL_DATA" - compose_cmd "${ROOT}/docker-compose/lakesoul/lakesoul.yaml" "" up -d - ## import tpch data into lakesoul - ## install rustup - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y - # shellcheck source=/dev/null - . "${HOME}/.cargo/env" - ## install rust nightly-2023-05-20 - rustup install nightly-2023-05-20 - ## download&generate tpch data - mkdir -p lakesoul/test_files/tpch/data - git clone https://github.com/databricks/tpch-dbgen.git - ( - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - mv *.tbl ../lakesoul/test_files/tpch/data - ) - export TPCH_DATA=$(realpath lakesoul/test_files/tpch/data) - ## import tpch data - git clone https://github.com/lakesoul-io/LakeSoul.git - # git checkout doris_dev - ( - cd LakeSoul/rust - cargo test load_tpch_data --package lakesoul-datafusion --features=ci -- --nocapture - ) - fi -} - dump_kerberos_container_state() { local container="$1" @@ -1982,10 +1943,6 @@ if [[ "${RUN_MARIADB}" -eq 1 ]]; then launch_component "mariadb" "${LOG_ROOT}/start_mariadb.log" start_mariadb fi -if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then - launch_component "lakesoul" "${LOG_ROOT}/start_lakesoul.log" start_lakesoul -fi - if [[ "${RUN_FLUSS}" -eq 1 ]]; then launch_component "fluss" "${LOG_ROOT}/start_fluss.log" start_fluss fi diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index 32ee285e2f7cca..7a50a02bf71037 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -128,27 +128,6 @@ under the License. test - - - ${project.groupId} - fe-filesystem-local - ${project.version} - test - - - - - commons-lang - commons-lang - test - ${project.groupId} fe-filesystem-hdfs-base diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java deleted file mode 100644 index 00b391ee3a18d8..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveAcidUtil.java +++ /dev/null @@ -1,486 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsAcidConstants; -import org.apache.doris.filesystem.FileEntry; -import org.apache.doris.filesystem.FileIterator; -import org.apache.doris.filesystem.FileSystem; -import org.apache.doris.filesystem.Location; - -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList.RangeResponse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Pure directory-name ACID state resolution for transactional Hive tables. - * - *

Plugin-side port of fe-core {@code AcidUtil.getAcidState}. It resolves, for one partition - * directory, the "best" base plus the working set of delta / delete-delta directories under a snapshot - * ({@code ValidTxnList} + {@code ValidWriteIdList}) and returns the surviving data files (base + - * non-delete deltas) plus the delete-delta descriptors. The BE later applies row deletes from the - * delete deltas.

- * - *

The fe-core version drags in {@code FileCacheValue}/{@code LocationPath}/{@code AcidInfo}/ - * {@code HivePartition}; those all drop out at the plugin boundary because the plugin lists files with the - * engine-injected Doris {@link FileSystem} and emits {@link HiveScanRange} directly. Only the pure name-parsing - * plus the {@code hive-common} {@code Valid*} algorithm ports.

- * - *

Ref: hive/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java#getAcidState (the fe-core copy - * exists because hive3 cannot read hive4 transaction tables and using hive4 directly is problematic).

- */ -public final class HiveAcidUtil { - private static final Logger LOG = LogManager.getLogger(HiveAcidUtil.class); - - private static final String HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX = "bucket_"; - private static final String DELTA_SIDE_FILE_SUFFIX = "_flush_length"; - - private HiveAcidUtil() { - } - - /** Resolved ACID state for one partition: surviving data files + delete-delta descriptors. */ - public static final class AcidState { - private final List dataFiles; - private final List deleteDeltas; - - AcidState(List dataFiles, List deleteDeltas) { - this.dataFiles = dataFiles; - this.deleteDeltas = deleteDeltas; - } - - /** Base + non-delete delta bucket files that survive the snapshot; each becomes a scan split. */ - public List getDataFiles() { - return dataFiles; - } - - /** Delete-delta directories (with the bucket file names inside) the BE must subtract. */ - public List getDeleteDeltas() { - return deleteDeltas; - } - } - - /** A single delete-delta directory plus the (filtered) delete file names inside it. */ - public static final class DeleteDelta { - private final String directoryLocation; - private final List fileNames; - - DeleteDelta(String directoryLocation, List fileNames) { - this.directoryLocation = directoryLocation; - this.fileNames = fileNames; - } - - public String getDirectoryLocation() { - return directoryLocation; - } - - public List getFileNames() { - return fileNames; - } - } - - private static final class ParsedBase { - private final long writeId; - private final long visibilityId; - - ParsedBase(long writeId, long visibilityId) { - this.writeId = writeId; - this.visibilityId = visibilityId; - } - } - - private static ParsedBase parseBase(String name) { - //format1 : base_writeId - //format2 : base_writeId_visibilityId detail: https://issues.apache.org/jira/browse/HIVE-20823 - name = name.substring("base_".length()); - int index = name.indexOf("_v"); - if (index == -1) { - return new ParsedBase(Long.parseLong(name), 0); - } - return new ParsedBase( - Long.parseLong(name.substring(0, index)), - Long.parseLong(name.substring(index + 2))); - } - - private static final class ParsedDelta implements Comparable { - private final long min; - private final long max; - private final String path; - private final int statementId; - private final boolean deleteDelta; - private final long visibilityId; - - ParsedDelta(long min, long max, String path, int statementId, - boolean deleteDelta, long visibilityId) { - this.min = min; - this.max = max; - this.path = path; - this.statementId = statementId; - this.deleteDelta = deleteDelta; - this.visibilityId = visibilityId; - } - - /* - * Smaller minWID orders first; - * If minWID is the same, larger maxWID orders first; - * Otherwise, sort by stmtID; files w/o stmtID orders first. - * - * Compactions (Major/Minor) merge deltas/bases but delete of old files - * happens in a different process; thus it's possible to have bases/deltas with - * overlapping writeId boundaries. The sort order helps figure out the "best" set of files - * to use to get data. - * This sorts "wider" delta before "narrower" i.e. delta_5_20 sorts before delta_5_10 (and delta_11_20) - */ - @Override - public int compareTo(ParsedDelta other) { - return min != other.min ? Long.compare(min, other.min) : - other.max != max ? Long.compare(other.max, max) : - statementId != other.statementId - ? Integer.compare(statementId, other.statementId) : - path.compareTo(other.path); - } - } - - private static boolean isValidMetaDataFile(FileSystem fileSystem, String baseDir) - throws IOException { - String fileLocation = baseDir + "_metadata_acid"; - try { - return fileSystem.exists(Location.of(fileLocation)); - } catch (IOException e) { - return false; - } - } - - private static boolean isValidBase(FileSystem fileSystem, String baseDir, - ParsedBase base, ValidWriteIdList writeIdList) throws IOException { - if (base.writeId == Long.MIN_VALUE) { - //Ref: https://issues.apache.org/jira/browse/HIVE-13369 - //such base is created by 1st compaction in case of non-acid to acid table conversion.(you - //will get dir: `base_-9223372036854775808`) - //By definition there are no open txns with id < 1. - //After this: https://issues.apache.org/jira/browse/HIVE-18192, txns(global transaction ID) => writeId. - return true; - } - - // hive 4 : just check "_v" suffix, before hive 4 : check `_metadata_acid` file in baseDir. - if ((base.visibilityId > 0) || isValidMetaDataFile(fileSystem, baseDir)) { - return writeIdList.isValidBase(base.writeId); - } - - // if here, it's a result of IOW - return writeIdList.isWriteIdValid(base.writeId); - } - - private static ParsedDelta parseDelta(String fileName, String deltaPrefix, String path) { - // format1: delta_min_max_statementId_visibilityId, delete_delta_min_max_statementId_visibilityId - // _visibilityId maybe not exists. - // detail: https://issues.apache.org/jira/browse/HIVE-20823 - // format2: delta_min_max_visibilityId, delete_delta_min_visibilityId - // when minor compaction runs, we collapse per statement delta files inside a single - // transaction so we no longer need a statementId in the file name - - long visibilityId = 0; - int visibilityIdx = fileName.indexOf("_v"); - if (visibilityIdx != -1) { - visibilityId = Long.parseLong(fileName.substring(visibilityIdx + 2)); - fileName = fileName.substring(0, visibilityIdx); - } - - boolean deleteDelta = deltaPrefix.equals("delete_delta_"); - - String rest = fileName.substring(deltaPrefix.length()); - int split = rest.indexOf('_'); - int split2 = rest.indexOf('_', split + 1); - long min = Long.parseLong(rest.substring(0, split)); - - if (split2 == -1) { - long max = Long.parseLong(rest.substring(split + 1)); - return new ParsedDelta(min, max, path, -1, deleteDelta, visibilityId); - } - - long max = Long.parseLong(rest.substring(split + 1, split2)); - int statementId = Integer.parseInt(rest.substring(split2 + 1)); - return new ParsedDelta(min, max, path, statementId, deleteDelta, visibilityId); - } - - private interface FileFilter { - boolean accept(String fileName); - } - - private static final class FullAcidFileFilter implements FileFilter { - @Override - public boolean accept(String fileName) { - return fileName.startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX) - && !fileName.endsWith(DELTA_SIDE_FILE_SUFFIX); - } - } - - private static final class InsertOnlyFileFilter implements FileFilter { - @Override - public boolean accept(String fileName) { - return true; - } - } - - /** - * Lists the immediate children (files and directories) of {@code dir}, non-recursively, via the - * engine-injected Doris {@link FileSystem#list} — the literal listing that mirrors the old - * {@code FileSystem.listStatus}. - * - *

Literal, not glob: uses {@code fs.list(loc)}, never {@code fs.listFiles(loc)}. The per-scheme - * filesystems ({@code DFSFileSystem}, {@code S3CompatibleFileSystem}) override {@code listFiles} with a - * glob-aware branch that would treat a location containing {@code [}/{@code *}/{@code ?} as a pattern; a hive - * partition/delta location can legitimately contain those characters, and the old {@code listStatus} never - * glob-expanded. Mirrors {@code HiveFileListingCache.listFromFileSystem}.

- */ - private static List listEntries(FileSystem fs, String dir) throws IOException { - List entries = new ArrayList<>(); - try (FileIterator it = fs.list(Location.of(dir))) { - while (it.hasNext()) { - entries.add(it.next()); - } - } - return entries; - } - - /** - * Lists the immediate file children of {@code dir} (directories excluded). - * - *

Mirrors fe-core {@code globList(fs, dir, false)} on a bare directory path: with no wildcard the - * glob has a {@code null} pattern, so it returns files only and skips any nested directory. The literal - * {@link #listEntries} returns both, so the directory entries are filtered out here.

- */ - private static List listFiles(FileSystem fs, String dir) throws IOException { - List files = new ArrayList<>(); - for (FileEntry entry : listEntries(fs, dir)) { - if (!entry.isDirectory()) { - files.add(entry); - } - } - return files; - } - - /** - * Resolves the ACID state of one partition directory under the given snapshot. - * - * @param fs engine-injected Doris file system for the partition location - * @param partitionPath the partition directory (e.g. {@code hdfs://.../data_id=200103}) - * @param txnValidIds the two serialized {@code Valid*} lists keyed by {@link HmsAcidConstants} - * @param isFullAcid full-ACID (bucket_ filter + delete deltas) vs insert-only (accept-all) - * @return surviving data files + delete-delta descriptors for the partition - */ - public static AcidState getAcidState(FileSystem fs, String partitionPath, - Map txnValidIds, boolean isFullAcid) throws IOException { - // Ref: https://issues.apache.org/jira/browse/HIVE-18192 - // Readers should use the combination of ValidTxnList and ValidWriteIdList(Table) for snapshot isolation. - // ValidReadTxnList implements ValidTxnList - // ValidReaderWriteIdList implements ValidWriteIdList - ValidTxnList validTxnList; - if (txnValidIds.containsKey(HmsAcidConstants.VALID_TXNS_KEY)) { - validTxnList = new ValidReadTxnList(); - validTxnList.readFromString(txnValidIds.get(HmsAcidConstants.VALID_TXNS_KEY)); - } else { - throw new RuntimeException("Miss ValidTxnList"); - } - - ValidWriteIdList validWriteIdList; - if (txnValidIds.containsKey(HmsAcidConstants.VALID_WRITEIDS_KEY)) { - validWriteIdList = new ValidReaderWriteIdList(); - validWriteIdList.readFromString(txnValidIds.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - } else { - throw new RuntimeException("Miss ValidWriteIdList"); - } - - //hdfs://xxxxx/user/hive/warehouse/username/data_id=200103 - // List all files and folders, without recursion. - List partitionEntries = listEntries(fs, partitionPath); - - String oldestBase = null; - long oldestBaseWriteId = Long.MAX_VALUE; - String bestBasePath = null; - long bestBaseWriteId = 0; - boolean haveOriginalFiles = false; - List workingDeltas = new ArrayList<>(); - - for (FileEntry entry : partitionEntries) { - if (entry.isDirectory()) { - String dirName = entry.name(); //dirName: base_xxx,delta_xxx,... - String dirPath = partitionPath + "/" + dirName; - - if (dirName.startsWith("base_")) { - ParsedBase base = parseBase(dirName); - if (!validTxnList.isTxnValid(base.visibilityId)) { - //checks visibilityTxnId to see if it is committed in current snapshot. - continue; - } - - long writeId = base.writeId; - if (oldestBaseWriteId > writeId) { - oldestBase = dirPath; - oldestBaseWriteId = writeId; - } - - if (((bestBasePath == null) || (bestBaseWriteId < writeId)) - && isValidBase(fs, dirPath, base, validWriteIdList)) { - //IOW will generator a base_N/ directory: https://issues.apache.org/jira/browse/HIVE-14988 - //So maybe need consider: https://issues.apache.org/jira/browse/HIVE-25777 - - bestBasePath = dirPath; - bestBaseWriteId = writeId; - } - } else if (dirName.startsWith("delta_") || dirName.startsWith("delete_delta_")) { - String deltaPrefix = dirName.startsWith("delta_") ? "delta_" : "delete_delta_"; - ParsedDelta delta = parseDelta(dirName, deltaPrefix, dirPath); - - if (!validTxnList.isTxnValid(delta.visibilityId)) { - continue; - } - - // No need check (validWriteIdList.isWriteIdRangeAborted(min,max) != RangeResponse.ALL) - // It is a subset of (validWriteIdList.isWriteIdRangeValid(min, max) != RangeResponse.NONE) - if (validWriteIdList.isWriteIdRangeValid(delta.min, delta.max) != RangeResponse.NONE) { - workingDeltas.add(delta); - } - } else { - //Sometimes hive will generate temporary directories(`.hive-staging_hive_xxx` ), - // which do not need to be read. - LOG.warn("Read Hive Acid Table ignore the contents of this folder:" + dirName); - } - } else { - haveOriginalFiles = true; - } - } - - if (bestBasePath == null && haveOriginalFiles) { - // ALTER TABLE nonAcidTbl SET TBLPROPERTIES ('transactional'='true'); - throw new UnsupportedOperationException("For no acid table convert to acid, please COMPACT 'major'."); - } - - if ((oldestBase != null) && (bestBasePath == null)) { - /* - * If here, it means there was a base_x (> 1 perhaps) but none were suitable for given - * {@link writeIdList}. Note that 'original' files are logically a base_Long.MIN_VALUE and thus - * cannot have any data for an open txn. We could check {@link deltas} has files to cover - * [1,n] w/o gaps but this would almost never happen... - * - * We only throw for base_x produced by Compactor since that base erases all history and - * cannot be used for a client that has a snapshot in which something inside this base is - * open. (Nor can we ignore this base of course) But base_x which is a result of IOW, - * contains all history so we treat it just like delta wrt visibility. Imagine, IOW which - * aborts. It creates a base_x, which can and should just be ignored.*/ - long[] exceptions = validWriteIdList.getInvalidWriteIds(); - String minOpenWriteId = ((exceptions != null) - && (exceptions.length > 0)) ? String.valueOf(exceptions[0]) : "x"; - throw new IOException( - String.format("Not enough history available for ({},{}). Oldest available base: {}", - validWriteIdList.getHighWatermark(), minOpenWriteId, oldestBase)); - } - - workingDeltas.sort(null); - - List deltas = new ArrayList<>(); - long current = bestBaseWriteId; - int lastStatementId = -1; - ParsedDelta prev = null; - // find need read delta/delete_delta file. - for (ParsedDelta next : workingDeltas) { - if (next.max > current) { - if (validWriteIdList.isWriteIdRangeValid(current + 1, next.max) != RangeResponse.NONE) { - deltas.add(next); - current = next.max; - lastStatementId = next.statementId; - prev = next; - } - } else if ((next.max == current) && (lastStatementId >= 0)) { - //make sure to get all deltas within a single transaction; multi-statement txn - //generate multiple delta files with the same txnId range - //of course, if maxWriteId has already been minor compacted, - // all per statement deltas are obsolete - - deltas.add(next); - prev = next; - } else if ((prev != null) - && (next.max == prev.max) - && (next.min == prev.min) - && (next.statementId == prev.statementId)) { - // The 'next' parsedDelta may have everything equal to the 'prev' parsedDelta, except - // the path. This may happen when we have split update and we have two types of delta - // directories- 'delta_x_y' and 'delete_delta_x_y' for the SAME txn range. - - // Also note that any delete_deltas in between a given delta_x_y range would be made - // obsolete. For example, a delta_30_50 would make delete_delta_40_40 obsolete. - // This is valid because minor compaction always compacts the normal deltas and the delete - // deltas for the same range. That is, if we had 3 directories, delta_30_30, - // delete_delta_40_40 and delta_50_50, then running minor compaction would produce - // delta_30_50 and delete_delta_30_50. - deltas.add(next); - prev = next; - } - } - - List dataFiles = new ArrayList<>(); - List deleteDeltas = new ArrayList<>(); - - FileFilter fileFilter = isFullAcid ? new FullAcidFileFilter() : new InsertOnlyFileFilter(); - - // delta directories - for (ParsedDelta delta : deltas) { - String location = delta.path; - List entries = listFiles(fs, location); - if (delta.deleteDelta) { - List deleteDeltaFileNames = new ArrayList<>(); - for (FileEntry entry : entries) { - String name = entry.name(); - if (fileFilter.accept(name)) { - deleteDeltaFileNames.add(name); - } - } - deleteDeltas.add(new DeleteDelta(location, deleteDeltaFileNames)); - continue; - } - for (FileEntry entry : entries) { - if (fileFilter.accept(entry.name())) { - dataFiles.add(entry); - } - } - } - - // base - if (bestBasePath != null) { - List entries = listFiles(fs, bestBasePath); - for (FileEntry entry : entries) { - if (fileFilter.accept(entry.name())) { - dataFiles.add(entry); - } - } - } - - if (!isFullAcid && !deleteDeltas.isEmpty()) { - throw new RuntimeException("No Hive Full Acid Table have delete_delta_* Dir."); - } - return new AcidState(dataFiles, deleteDeltas); - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 7b6c5656806bfe..013db6a79f0522 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -96,12 +96,8 @@ public class HiveConnector implements Connector { private volatile HadoopAuthenticator pluginAuth; private volatile boolean pluginAuthComputed; - // Read-transaction manager for transactional (ACID) Hive scans. One per connector, keyed by query. - // Plugin-owned; live since the read cutover wired its query-finish commit (see the manager). - private final HiveReadTransactionManager readTxnManager = new HiveReadTransactionManager(); - // Connector-owned directory-listing cache, shared by the (per-call) scan provider and metadata. One per - // connector (like readTxnManager above) — the scan provider / metadata are rebuilt per query, so the cache + // connector — the scan provider / metadata are rebuilt per query, so the cache // must live on the long-lived connector to survive across scans. Built from catalog props. Its // metastore-metadata sibling is the CachingHmsClient wrapping the HmsClient. private final HiveFileListingCache fileListingCache; @@ -256,7 +252,7 @@ private Connector resolveSiblingOwner(ConnectorTableHandle handle) { @Override public ConnectorScanPlanProvider getScanPlanProvider() { - return new HiveScanPlanProvider(getOrCreateClient(), props, context, readTxnManager, fileListingCache); + return new HiveScanPlanProvider(getOrCreateClient(), props, context, fileListingCache); } /** diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java deleted file mode 100644 index 2c33edb961e13a..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransaction.java +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsClient; -import org.apache.doris.connector.spi.DorisConnectorException; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Holds the state of one Hive read transaction, used when reading a transactional (ACID) Hive table. - * Each instance is bound to a single query. - * - *

Plugin-side port of fe-core {@code HiveTransaction}. It drives the four read-side metastore - * primitives through the shared {@link HmsClient} ({@code openTxn} / {@code acquireSharedLock} / - * {@code getValidWriteIds} / {@code commitTxn}) instead of the fe-core {@code HMSExternalCatalog}/ - * {@code HMSCachedClient}/{@code TableNameInfo} chain — the table identity is carried as plain - * {@code (dbName, tableName)} strings.

- */ -public class HiveReadTransaction { - private final String queryId; - private final String user; - private final String dbName; - private final String tableName; - private final boolean isFullAcid; - private final HmsClient hmsClient; - - private long txnId; - private final List partitionNames = new ArrayList<>(); - - private Map txnValidIds = null; - - public HiveReadTransaction(String queryId, String user, String dbName, String tableName, - boolean isFullAcid, HmsClient hmsClient) { - this.queryId = queryId; - this.user = user; - this.dbName = dbName; - this.tableName = tableName; - this.isFullAcid = isFullAcid; - this.hmsClient = hmsClient; - } - - public String getQueryId() { - return queryId; - } - - public void addPartition(String partitionName) { - this.partitionNames.add(partitionName); - } - - public boolean isFullAcid() { - return isFullAcid; - } - - /** - * Acquires a shared read lock and fetches the snapshot ({@code ValidTxnList} + {@code - * ValidWriteIdList}) for the current transaction, memoizing so the lock is taken at most once. The - * lock is released when the transaction is committed at query finish. - */ - public Map getValidWriteIds() { - if (txnValidIds == null) { - hmsClient.acquireSharedLock(queryId, txnId, user, dbName, tableName, partitionNames, 5000); - txnValidIds = hmsClient.getValidWriteIds(dbName + "." + tableName, txnId); - } - return txnValidIds; - } - - public void begin() { - try { - this.txnId = hmsClient.openTxn(user); - } catch (RuntimeException e) { - throw new DorisConnectorException(e.getMessage(), e); - } - } - - public void commit() { - try { - hmsClient.commitTxn(txnId); - } catch (RuntimeException e) { - throw new DorisConnectorException(e.getMessage(), e); - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java deleted file mode 100644 index 8ad06a6a776eb8..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveReadTransactionManager.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Manages the read transaction of each query against transactional (ACID) Hive tables. For each query a - * {@link HiveReadTransaction} is registered (which opens the metastore transaction); when the query - * finishes it is deregistered (which commits the transaction, releasing the shared read lock). - * - *

Plugin-side port of fe-core {@code HiveTransactionMgr}. Plugin-owned and live since the read - * cutover: a transactional-hive scan registers via {@code HiveScanPlanProvider.planAcidScan}, and the - * fe-core query-finish callback ({@code PluginDrivenScanNode}) drives {@link #deregister} through - * {@code releaseReadTransaction}. The legacy fe-core {@code HiveScanNode} / - * {@code Env.getCurrentHiveTransactionMgr()} are removed.

- */ -public class HiveReadTransactionManager { - private static final Logger LOG = LogManager.getLogger(HiveReadTransactionManager.class); - - private final Map txnMap = new ConcurrentHashMap<>(); - - /** - * Opens the transaction (see {@link HiveReadTransaction#begin}) and registers it under its query id, so - * the query-finish path can find and commit it. Mirrors fe-core: one {@code HiveReadTransaction} is - * created per transactional-table scan (each pins its own table's write-id snapshot), keyed by query id. - */ - public void register(HiveReadTransaction txn) { - txn.begin(); - txnMap.put(txn.getQueryId(), txn); - } - - public void deregister(String queryId) { - HiveReadTransaction txn = txnMap.remove(queryId); - if (txn != null) { - try { - txn.commit(); - } catch (RuntimeException e) { - LOG.warn("failed to commit hive read txn: " + queryId, e); - } - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java index 18c4b142adc4ef..94b8edd86d306e 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java @@ -35,18 +35,14 @@ import org.apache.doris.connector.spi.scan.ConnectorScanRequest; import org.apache.doris.connector.spi.scan.ConnectorSplitSource; import org.apache.doris.connector.spi.scan.ScanNodePropertyKeys; -import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileSystem; import org.apache.doris.foundation.format.FileCompression; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileScanRangeParams; -import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -61,7 +57,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.UnaryOperator; /** * Scan plan provider for Hive tables. @@ -74,8 +69,7 @@ *
  • Return {@link HiveScanRange} for each file split
  • * * - *

    Non-transactional tables can stream file splits without materializing a partition's ranges. - * Transactional tables retain snapshot-aware synchronous planning.

    + *

    Tables can stream file splits without materializing a partition's ranges.

    */ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { @@ -89,12 +83,6 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { private static final int MAX_PARTITIONS = 100000; private static final int DEFAULT_BATCH_THRESHOLD = 1024; - /** - * Connector-internal signal (consumed by {@link #populateScanLevelParams}) marking a full-ACID - * (transactional) Hive scan; drives the scan-level {@code table_format_type} stamp. Not a BE-facing key. - */ - public static final String PROP_TRANSACTIONAL_HIVE = "transactional_hive"; - /** * Session variable gating the OpenX-JSON "read the whole JSON row into one CSV column" mode (legacy * SessionVariable.read_hive_json_in_one_column). Byte-identical to the fe-core session-var name; it is @@ -103,29 +91,22 @@ public class HiveScanPlanProvider implements ConnectorScanPlanProvider { private static final String SESSION_READ_HIVE_JSON_IN_ONE_COLUMN = "read_hive_json_in_one_column"; static final String SCAN_REUSE_NAMESPACE = "hms.scan-reuse"; - /** Input format of a full-ACID (ORC) transactional Hive table; other formats are rejected. */ - private static final String ORC_ACID_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; - private final HmsClient hmsClient; private final HiveCatalogProperties catalogProperties; - // Engine-owned, per-catalog filesystem accessor. The non-ACID listing path borrows the Doris FileSystem via + // Engine-owned, per-catalog filesystem accessor. The listing path borrows the Doris FileSystem via // storage().getFileSystem(session) (never closes it — the engine owns its lifecycle) to list partition // directories, replacing bare Hadoop FileSystem.get (FIX-HIVEFS: the hive plugin bundles no HDFS impl). private final ConnectorContext context; - private final HiveReadTransactionManager readTxnManager; // Connector-owned directory-listing cache (the SAME instance HiveConnector shares with HiveConnectorMetadata), - // so a repeated scan of the same partition directory is served from the cache instead of re-listing. Only the - // plain (non-ACID) path uses it; the ACID path lists via HiveAcidUtil and is uncached (legacy parity). + // so a repeated scan of the same partition directory is served from the cache instead of re-listing. private final HiveFileListingCache fileListingCache; private final PartitionBatchProfile partitionBatchProfile = new PartitionBatchProfile(); public HiveScanPlanProvider(HmsClient hmsClient, HiveCatalogProperties catalogProperties, - ConnectorContext context, HiveReadTransactionManager readTxnManager, - HiveFileListingCache fileListingCache) { + ConnectorContext context, HiveFileListingCache fileListingCache) { this.hmsClient = hmsClient; this.catalogProperties = catalogProperties; this.context = context; - this.readTxnManager = readTxnManager; this.fileListingCache = fileListingCache; } @@ -143,14 +124,10 @@ public boolean usesHiveParquetInt96TimeZone() { @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); + rejectTransactionalRead(hiveHandle); if (session == null || !session.isExternalScanTaskReuseEnabled()) { return doPlanScan(session, request); } - if (hiveHandle.isTransactional()) { - // ACID / INSERT_ONLY reads open a per-scan read transaction with a write-id snapshot and - // a shared metastore lock; reusing the planned ranges would skip that transaction. - return doPlanScan(session, request); - } // Statement-scoped reuse: within one statement the identical scan (same table, same // partition set, same formats) plans once and every duplicated relation shares the result. // The scope is NONE for offline planning and tests, in which case the loader runs on every @@ -199,27 +176,18 @@ private List doPlanScan(ConnectorSession session, ConnectorS boolean isLzo = isLzoInputFormat(hiveHandle.getInputFormat()); // LZO text is NOT splittable: a .lzo stream cannot be decompressed from an arbitrary byte offset. // Legacy HiveUtil.isSplittable returned false for LZO; HiveFileFormat maps LZO text to TEXT (which - // reports splittable), so the LZO case must be masked out here. ACID is never LZO, so this leaves the - // transactional branch unchanged. + // reports splittable), so the LZO case must be masked out here. boolean splittable = fileFormat.isSplittable() && !isLzo; List ranges = new ArrayList<>(); - if (hiveHandle.isTransactional()) { - // Transactional (ACID) table: descend into base/delta directories under the query's write-id - // snapshot and emit ACID-annotated ranges. Borrows the engine's per-catalog Doris FileSystem to - // list (same source as the non-ACID branch below; the engine owns its lifecycle — never closed here). - planAcidScan(session, hiveHandle, partitions, storage().getFileSystem(session), fileFormat, - splittable, targetSplitSize, ranges); - } else { - // Borrow the engine's per-catalog Doris FileSystem to list partition directories (see field javadoc). - FileSystem fs = storage().getFileSystem(session); - for (PartitionScanInfo partition : partitions) { - HiveFileFormat partFormat = partition.fileFormat != null - ? partition.fileFormat : fileFormat; - listAndSplitFiles(dbName, tableName, partition, partFormat, - splittable, isLzo, targetSplitSize, fs, ranges, completeness); - } + // Borrow the engine's per-catalog Doris FileSystem to list partition directories (see field javadoc). + FileSystem fs = storage().getFileSystem(session); + for (PartitionScanInfo partition : partitions) { + HiveFileFormat partFormat = partition.fileFormat != null + ? partition.fileFormat : fileFormat; + listAndSplitFiles(dbName, tableName, partition, partFormat, + splittable, isLzo, targetSplitSize, fs, ranges, completeness); } LOG.info("Hive scan plan: table={}.{}, partitions={}, splits={}", @@ -235,25 +203,21 @@ private List doPlanScan(ConnectorSession session, ConnectorS * partition's files synchronously (legacy {@code HiveScanNode} went async at * {@code num_partitions_in_batch_mode}). * - *

    Transactional/ACID tables are excluded DELIBERATELY: their scan opens one metastore read transaction - * (see {@link #planAcidScan}); running the per-batch resolution on background threads would open — and leak - * — a read transaction per batch. ACID partitioned tables keep the synchronous {@link #planScan} path - * (correct, just not streamed). The transactional test uses the SAME {@code isTransactional()} accessor - * {@link #planScan} branches on.

    */ @Override public boolean supportsBatchScan(ConnectorSession session, ConnectorTableHandle handle) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; + rejectTransactionalRead(hiveHandle); List partKeyNames = hiveHandle.getPartitionKeyNames(); - return partKeyNames != null && !partKeyNames.isEmpty() && !hiveHandle.isTransactional(); + return partKeyNames != null && !partKeyNames.isEmpty(); } @Override public long streamingSplitEstimate(ConnectorSession session, ConnectorTableHandle handle, Optional filter, boolean countPushdown) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; - if (hiveHandle.isTransactional() - || Boolean.FALSE.equals(session.getProperty("enable_external_table_batch_mode", Boolean.class))) { + rejectTransactionalRead(hiveHandle); + if (Boolean.FALSE.equals(session.getProperty("enable_external_table_batch_mode", Boolean.class))) { return -1; } int fileThreshold = sessionInt(session, "num_files_in_batch_mode", DEFAULT_BATCH_THRESHOLD); @@ -291,12 +255,18 @@ private long cachedFileCount(HiveTableHandle handle) { @Override public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorScanRequest request) { HiveTableHandle hiveHandle = (HiveTableHandle) request.getTableHandle(); - if (hiveHandle.isTransactional()) { - throw new DorisConnectorException("Transactional Hive scans require snapshot-aware planning"); - } + rejectTransactionalRead(hiveHandle); return new HiveSplitSource(session, hiveHandle, request.getRequiredPartitions()); } + private static void rejectTransactionalRead(HiveTableHandle handle) { + // Both full ACID and insert-only layouts need write-id visibility; plain file scans + // (including metadata-only counts) cannot return a transactionally correct result. + if (handle.isTransactional()) { + throw new DorisConnectorException("Hive transactional table reads are no longer supported"); + } + } + private static int sessionInt(ConnectorSession session, String name, int defaultValue) { Integer value = session.getProperty(name, Integer.class); return value == null ? defaultValue : value; @@ -320,6 +290,7 @@ private final class HiveSplitSource implements ConnectorSplitSource { private volatile boolean closed; HiveSplitSource(ConnectorSession session, HiveTableHandle handle, List requiredPartitions) { + rejectTransactionalRead(handle); this.handle = handle; this.partitionNames = requiredPartitions.isEmpty() ? null : requiredPartitions; this.fs = storage().getFileSystem(session); @@ -361,7 +332,7 @@ private void loadNextFileRanges() { } HiveFileFormat format = partition.fileFormat == null ? tableFormat : partition.fileFormat; ranges = splitFileRanges(file.getPath(), file.getLength(), file.getModificationTime(), - partition, format, splittable, targetSplitSize, null, null); + partition, format, splittable, targetSplitSize); } private void loadNextPartitionFiles() { @@ -452,8 +423,7 @@ public TFileCompressType adjustFileCompressType(TFileCompressType inferred) { * {@link #planScan} per batch) would emit every partition's files once per batch — duplicate rows. The batch * strings are the metastore-rendered {@code key=value/...} partition names (the keys of the Nereids * selected-partition map), exactly the form {@code hmsClient.getPartitions} accepts, so batch-scoped - * resolution is a clean {@code getPartitions(batch)} round-trip. Only the non-ACID path is reachable here - * ({@link #supportsBatchScan} excludes transactional tables). Collects the same partition-scoped source + * resolution is a clean {@code getPartitions(batch)} round-trip. Collects the same partition-scoped source * that {@link #streamSplits} consumes lazily, preserving its file filtering and split boundaries. */ @Override @@ -478,87 +448,6 @@ public List planScanForPartitionBatch( return ranges; } - /** - * Plans a scan of a transactional (ACID) Hive table. - * - *

    Opens (or reuses) the query's read transaction — which acquires a shared read lock and pins a - * write-id snapshot — then, per partition, runs {@link HiveAcidUtil#getAcidState} to resolve the - * surviving base/delta data files and delete-delta directories, and emits one ACID-annotated - * {@link HiveScanRange} per data-file split. The BE subtracts the delete deltas on read.

    - * - *

    Live path. Post-cutover an {@code hms} catalog is plugin-driven, so an hms-catalog - * transactional table is a {@code PluginDrivenExternalTable} routed through {@code PluginDrivenScanNode} - * straight into this method on a live query (the only read gate is the full-ACID ORC-format check below). - * It opens a real metastore transaction/lock; the matching commit (lock release) is driven by - * {@link HiveReadTransactionManager#deregister} at query finish (via {@link #releaseReadTransaction}), - * without which the shared read lock would leak for the metastore's lifetime.

    - */ - private void planAcidScan(ConnectorSession session, HiveTableHandle handle, - List partitions, FileSystem fs, HiveFileFormat fileFormat, - boolean splittable, long targetSplitSize, List ranges) { - boolean isFullAcid = handle.isFullAcid(); - if (isFullAcid && !ORC_ACID_INPUT_FORMAT.equals(handle.getInputFormat())) { - // Full-ACID data is only stored in the ORC ACID layout; reject other formats loudly - // (legacy parity: HMSExternalTable.isFullAcidTable throws "no Orc Format"). - throw new DorisConnectorException("This table is full Acid Table, but no Orc Format."); - } - - // One transaction per transactional-table scan, pinning this table's write-id snapshot (mirrors - // fe-core, which opens a HiveTransaction per scan node). The manager holds it for commit at finish. - HiveReadTransaction txn = new HiveReadTransaction(session.getQueryId(), session.getUser(), - handle.getDbName(), handle.getTableName(), isFullAcid, hmsClient); - readTxnManager.register(txn); - for (PartitionScanInfo partition : partitions) { - if (!partition.partitionValues.isEmpty()) { - txn.addPartition(buildPartitionName(partition.partitionValues)); - } - } - Map txnValidIds = txn.getValidWriteIds(); - - for (PartitionScanInfo partition : partitions) { - HiveFileFormat partFormat = partition.fileFormat != null - ? partition.fileFormat : fileFormat; - try { - // Descend the ACID directory tree through the engine-injected Doris FileSystem. The hive plugin - // bundles no HDFS impl, so a bare Hadoop FileSystem.get here would throw "No FileSystem for - // scheme hdfs" (FIX-HIVEFS); the engine owns this FileSystem's lifecycle — never closed here. - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - fs, partition.location, txnValidIds, isFullAcid); - // Only full-ACID reads are marked transactional_hive (delete deltas applied by the BE - // merge-on-read reader). Insert-only tables use the write-id snapshot to pick files but - // store plain data files, so their splits stay plain hive — matching legacy AcidUtil, - // which sets acidInfo only when isFullAcid. - // BE-facing ACID paths (delete-delta dir + partition location) also carry the raw HMS scheme; the - // BE transactional reader opens them via the same native S3 factory, so normalize s3a://->s3://. - String acidLocation = isFullAcid ? normalizeNativeUri(partition.location) : null; - List encodedDeltas = isFullAcid - ? encodeDeleteDeltas(state.getDeleteDeltas(), this::normalizeNativeUri) : null; - for (FileEntry dataFile : state.getDataFiles()) { - splitFile(dataFile.location().uri(), dataFile.length(), - dataFile.modificationTime(), partition, partFormat, splittable, - targetSplitSize, acidLocation, encodedDeltas, ranges); - } - } catch (IOException e) { - throw new DorisConnectorException( - "Failed to list ACID files for partition: " + partition.location, e); - } - } - } - - /** - * Commits and deregisters this query's read transaction (opened by {@link #planAcidScan} via - * {@link HiveReadTransactionManager#register}), releasing the metastore shared read lock. Driven by the - * generic fe-core query-finish callback at query end; without it a transactional-hive read leaks the shared - * read lock for the metastore's lifetime. {@code readTxnManager} is the same per-connector manager that - * {@code register} used (both injected by {@code HiveConnector}), so the {@code queryId} keys match. - * {@code deregister} is idempotent (a no-op for a query that opened no transaction) and swallows a commit - * failure, matching the best-effort SPI contract. - */ - @Override - public void releaseReadTransaction(String queryId) { - readTxnManager.deregister(queryId); - } - @Override public List collectScanProfiles(ConnectorSession session) { List profiles = new ArrayList<>(partitionBatchProfile.drain()); @@ -579,31 +468,6 @@ private static PartitionBatchProfile statementPruningFailureProfile(ConnectorSes return session.getStatementScope().computeIfAbsent(key, PartitionBatchProfile::new); } - /** Encodes each delete-delta as {@code "dir|file1,file2"} for {@link HiveScanRange.Builder#acidInfo}. */ - private static List encodeDeleteDeltas(List deltas, - UnaryOperator nativePathNormalizer) { - List encoded = new ArrayList<>(deltas.size()); - for (HiveAcidUtil.DeleteDelta delta : deltas) { - // Normalize the delete-delta directory scheme (s3a://->s3://) for BE's native reader; the file names - // are bare names appended after '|'. - encoded.add(nativePathNormalizer.apply(delta.getDirectoryLocation()) + "|" - + String.join(",", delta.getFileNames())); - } - return encoded; - } - - /** Builds the Hive partition name {@code k1=v1/k2=v2} used for the shared read lock components. */ - private static String buildPartitionName(Map partitionValues) { - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : partitionValues.entrySet()) { - if (sb.length() > 0) { - sb.append('/'); - } - sb.append(entry.getKey()).append('=').append(entry.getValue()); - } - return sb.toString(); - } - @Override public Map getScanNodeProperties( ConnectorSession session, @@ -611,6 +475,7 @@ public Map getScanNodeProperties( List columns, Optional filter) { HiveTableHandle hiveHandle = (HiveTableHandle) handle; + rejectTransactionalRead(hiveHandle); Map props = new HashMap<>(); // File format type @@ -658,39 +523,9 @@ public Map getScanNodeProperties( props.putAll(textProps); } - // #65437: BE selects the file scanner from SCAN-LEVEL params before the per-range splits arrive, and its - // FileScannerV2 gate excludes scans whose scan-level table_format_type is "transactional_hive" (V2 does - // not apply ACID delete deltas). Signal a full-ACID scan so populateScanLevelParams stamps it at scan - // level; gate on isFullAcid() to match the per-range marker (insert-only tables keep the V2 fast path). - if (hiveHandle.isFullAcid()) { - props.put(PROP_TRANSACTIONAL_HIVE, "true"); - } - return props; } - /** - * Stamps the SCAN-LEVEL table format for a full-ACID Hive scan (signalled by {@link #PROP_TRANSACTIONAL_HIVE} - * from {@link #getScanNodeProperties}). BE's FileScannerV2 selection reads {@code table_format_type} from the - * scan-level params (before per-range splits are fetched); without this stamp a transactional Hive read would - * wrongly use FileScannerV2, which does not apply ACID delete deltas — deleted rows would reappear. Mirrors the - * legacy {@code HiveScanNode.markTransactionalHiveScanParams} (#65437); the generic {@code PluginDrivenScanNode} - * invokes this hook after building the scan-level params. - */ - @Override - public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { - if ("true".equals(nodeProperties.get(PROP_TRANSACTIONAL_HIVE))) { - markTransactionalHiveScanParams(params); - } - } - - static void markTransactionalHiveScanParams(TFileScanRangeParams params) { - TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); - // Same literal the per-range marker uses (HiveScanRange); BE matches table_format_type == "transactional_hive". - tableFormatFileDesc.setTableFormatType("transactional_hive"); - params.setTableFormatParams(tableFormatFileDesc); - } - /** * Reads the {@code read_hive_json_in_one_column} session flag (default false) from the connector session — * the same channel the write path uses for {@code hive_text_compression}. The engine dumps every visible @@ -792,7 +627,7 @@ private void listAndSplitFiles(String dbName, String tableName, continue; } splitFile(file.getPath(), file.getLength(), file.getModificationTime(), - partition, fileFormat, splittable, targetSplitSize, null, null, ranges); + partition, fileFormat, splittable, targetSplitSize, ranges); } } @@ -891,21 +726,16 @@ private String normalizeNativeUri(String rawUri) { /** * Splits a file into scan ranges based on target split size. - * - *

    When {@code acidPartitionLocation} is non-null the ranges carry ACID delete-delta info - * (marking them {@code transactional_hive}); otherwise they are plain Hive ranges.

    */ private void splitFile(String filePath, long fileSize, long modTime, PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, - String acidPartitionLocation, List acidDeltas, List ranges) { splitFileRanges(filePath, fileSize, modTime, partition, fileFormat, splittable, - targetSplitSize, acidPartitionLocation, acidDeltas).forEachRemaining(ranges::add); + targetSplitSize).forEachRemaining(ranges::add); } private Iterator splitFileRanges(String filePath, long fileSize, long modTime, - PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, - String acidPartitionLocation, List acidDeltas) { + PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize) { if (fileSize == 0) { return Collections.emptyIterator(); } @@ -931,7 +761,7 @@ public ConnectorScanRange next() { } long length = Math.min(splitSize, fileSize - offset); ConnectorScanRange range = newRangeBuilder(normalizedPath, offset, length, fileSize, - modTime, fileFormat, partition, acidPartitionLocation, acidDeltas).build(); + modTime, fileFormat, partition).build(); offset += length; return range; } @@ -939,8 +769,7 @@ public ConnectorScanRange next() { } private static HiveScanRange.Builder newRangeBuilder(String filePath, long start, long length, - long fileSize, long modTime, HiveFileFormat fileFormat, PartitionScanInfo partition, - String acidPartitionLocation, List acidDeltas) { + long fileSize, long modTime, HiveFileFormat fileFormat, PartitionScanInfo partition) { HiveScanRange.Builder builder = HiveScanRange.builder() .path(filePath) .start(start) @@ -950,10 +779,6 @@ private static HiveScanRange.Builder newRangeBuilder(String filePath, long start .fileFormat(fileFormat.getFormatName()) .tableFormatType("hive") .partitionValues(partition.partitionValues); - if (acidPartitionLocation != null) { - // Sets tableFormatType="transactional_hive" and attaches the delete-delta descriptors. - builder.acidInfo(acidPartitionLocation, acidDeltas); - } return builder; } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java index 9e761832669436..6995ed4366cc2c 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanRange.java @@ -21,13 +21,9 @@ import org.apache.doris.connector.spi.scan.ConnectorScanRange; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TTableFormatFileDesc; -import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; -import org.apache.doris.thrift.TTransactionalHiveDesc; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,9 +35,6 @@ * Unlike JNI-based scan ranges (ES, JDBC, MaxCompute), Hive scan ranges * carry real file paths with byte offsets, native file format types * (PARQUET/ORC/TEXT), and partition values.

    - * - *

    For ACID tables, additional properties carry transaction info - * (partition location, delete delta paths).

    */ public class HiveScanRange implements ConnectorScanRange { @@ -137,11 +130,6 @@ public String toString() { @Override public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { - if ("transactional_hive".equals(getTableFormatType())) { - populateTransactionalHiveParams(formatDesc); - } - // Non-transactional hive needs no per-split TTableFormatFileDesc fields. - // Rewrite columns-from-path from the connector's partition values, mirroring // IcebergScanRange/PaimonScanRange. This connector now OWNS the Hive default-partition // sentinel (__HIVE_DEFAULT_PARTITION__ -> SQL NULL) mapping that fe-core's @@ -177,46 +165,6 @@ public void populateRangeParams(TTableFormatFileDesc formatDesc, } } - private void populateTransactionalHiveParams(TTableFormatFileDesc formatDesc) { - Map props = getProperties(); - TTransactionalHiveDesc txnDesc = new TTransactionalHiveDesc(); - - String partLoc = props.get("acid.partition_location"); - if (partLoc != null) { - txnDesc.setPartition(partLoc); - } - - String countStr = props.get("acid.delete_delta_count"); - if (countStr != null) { - int count = Integer.parseInt(countStr); - List deltas = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - String deltaStr = props.get("acid.delete_delta." + i); - if (deltaStr != null) { - TTransactionalHiveDeleteDeltaDesc delta = - new TTransactionalHiveDeleteDeltaDesc(); - // Encoded as "dir|file1,file2" (see Builder#acidInfo). BE needs BOTH the - // delete-delta directory AND the file names to correctly apply row deletes; - // dropping the file names silently under-deletes on ACID reads. - int sep = deltaStr.indexOf('|'); - if (sep >= 0) { - delta.setDirectoryLocation(deltaStr.substring(0, sep)); - String fileNamesPart = deltaStr.substring(sep + 1); - if (!fileNamesPart.isEmpty()) { - delta.setFileNames(Arrays.asList(fileNamesPart.split(","))); - } - } else { - delta.setDirectoryLocation(deltaStr); - } - deltas.add(delta); - } - } - txnDesc.setDeleteDeltas(deltas); - } - - formatDesc.setTransactionalHiveParams(txnDesc); - } - public static Builder builder() { return new Builder(); } @@ -289,28 +237,6 @@ public Builder properties(Map val) { return this; } - /** - * Adds ACID-related properties for transactional Hive tables. - * - * @param partitionLocation the partition directory path - * @param deleteDeltas list of delete delta descriptors (dir|file1,file2) - */ - public Builder acidInfo(String partitionLocation, List deleteDeltas) { - if (this.properties == null) { - this.properties = new HashMap<>(); - } - this.tableFormatType = "transactional_hive"; - this.properties.put("acid.partition_location", partitionLocation); - if (deleteDeltas != null) { - this.properties.put("acid.delete_delta_count", - String.valueOf(deleteDeltas.size())); - for (int i = 0; i < deleteDeltas.size(); i++) { - this.properties.put("acid.delete_delta." + i, deleteDeltas.get(i)); - } - } - return this; - } - public HiveScanRange build() { return new HiveScanRange(this); } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java deleted file mode 100644 index 7be2319459b6a1..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveAcidUtilTest.java +++ /dev/null @@ -1,263 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsAcidConstants; -import org.apache.doris.filesystem.FileEntry; -import org.apache.doris.filesystem.FileSystem; -import org.apache.doris.filesystem.Location; -import org.apache.doris.filesystem.local.LocalFileSystem; - -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.util.BitSet; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Tests the pure ACID directory descent {@link HiveAcidUtil#getAcidState} against a real Doris - * {@link FileSystem} (the test-only {@link LocalFileSystem}) over a local temp tree — the Doris equivalent of the - * old Hadoop {@code LocalFileSystem}, now that {@link HiveAcidUtil} lists via the engine-injected Doris filesystem. - * - *

    WHY: for a transactional Hive table the reader must reconstruct the correct snapshot from the - * base/delta/delete-delta directory layout — pick the best valid base, layer on the deltas whose - * write-id range is still valid, drop obsolete/out-of-snapshot directories, and hand the BE the - * delete-delta file names so it can subtract deleted rows. A wrong base or a dropped delta silently - * returns stale or over-/under-deleted data. These tests pin that selection algorithm.

    - * - *

    The filesystem is a {@link LiteralListingLocalFileSystem}: it forbids the glob-aware - * {@link FileSystem#listFiles}, so every test here also pins that {@link HiveAcidUtil} lists via the literal - * {@link FileSystem#list} (matching the old {@code FileSystem.listStatus}).

    - */ -public class HiveAcidUtilTest { - - @TempDir - java.nio.file.Path tempDir; - - private FileSystem localFs() { - return new LiteralListingLocalFileSystem(); - } - - /** Creates {@code //} with 1 byte of content. */ - private void createBucketFile(String dir, String fileName) throws IOException { - java.nio.file.Path d = tempDir.resolve(dir); - Files.createDirectories(d); - Files.write(d.resolve(fileName), new byte[] {1}); - } - - /** Snapshot with all visibility txns valid and write-ids valid up to {@code highWatermark}. */ - private Map snapshot(long highWatermark) { - ValidTxnList validTxnList = - new ValidReadTxnList(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); - ValidWriteIdList validWriteIdList = - new ValidReaderWriteIdList("db.tbl", new long[0], new BitSet(), highWatermark); - Map ids = new HashMap<>(); - ids.put(HmsAcidConstants.VALID_TXNS_KEY, validTxnList.writeToString()); - ids.put(HmsAcidConstants.VALID_WRITEIDS_KEY, validWriteIdList.writeToString()); - return ids; - } - - @Test - public void testBestBaseWithDeltaAndDeleteDelta() throws IOException { - // Best base = base_5 (base_2 superseded); delta_6 layered on; delete_delta_6 survives via the - // split-update pairing with delta_6; delta_9 is out of the write-id=6 snapshot; the staging dir - // and the _flush_length side file are ignored. - createBucketFile("base_0000005", "bucket_00000"); - createBucketFile("base_0000002", "bucket_00000"); - createBucketFile("delta_0000006_0000006", "bucket_00000"); - createBucketFile("delta_0000006_0000006", "bucket_00000_flush_length"); - createBucketFile("delete_delta_0000006_0000006", "bucket_00000"); - createBucketFile("delta_0000009_0000009", "bucket_00000"); - createBucketFile(".hive-staging_hive_2026-01-01", "junk"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(6L), true); - - List dataFiles = state.getDataFiles(); - Assertions.assertEquals(2, dataFiles.size(), - "surviving data files must be exactly base_5 + delta_6 bucket files"); - boolean hasBase5 = false; - boolean hasDelta6 = false; - for (FileEntry f : dataFiles) { - String p = f.location().uri(); - Assertions.assertFalse(p.contains("base_0000002"), "superseded base must be dropped: " + p); - Assertions.assertFalse(p.contains("delta_0000009"), "out-of-snapshot delta dropped: " + p); - Assertions.assertFalse(p.endsWith("_flush_length"), "side file excluded: " + p); - hasBase5 |= p.contains("/base_0000005/bucket_00000"); - hasDelta6 |= p.contains("/delta_0000006_0000006/bucket_00000"); - } - Assertions.assertTrue(hasBase5, "best base_5 bucket file must survive"); - Assertions.assertTrue(hasDelta6, "delta_6 bucket file must survive"); - - List deletes = state.getDeleteDeltas(); - Assertions.assertEquals(1, deletes.size(), "the paired delete_delta_6 must survive"); - HiveAcidUtil.DeleteDelta d = deletes.get(0); - Assertions.assertTrue(d.getDirectoryLocation().endsWith("/delete_delta_0000006_0000006"), - "delete-delta dir: " + d.getDirectoryLocation()); - Assertions.assertEquals(List.of("bucket_00000"), d.getFileNames(), - "delete-delta file names must be captured (BE under-deletes without them)"); - } - - @Test - public void testInsertOnlyRejectsDeleteDelta() throws IOException { - // An insert-only ACID table must never carry delete deltas; a stray one is a corruption signal. - createBucketFile("base_0000005", "000000_0"); - createBucketFile("delta_0000006_0000006", "000000_0"); - createBucketFile("delete_delta_0000006_0000006", "000000_0"); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(6L), false)); - Assertions.assertTrue(ex.getMessage().contains("delete_delta"), - "insert-only table with a delete delta must fail loud: " + ex.getMessage()); - } - - @Test - public void testInsertOnlyAcceptsAllFileNames() throws IOException { - // Insert-only uses the accept-all filter: files that are not bucket_* still count as data. - createBucketFile("base_0000005", "000000_0"); - createBucketFile("delta_0000006_0000006", "000001_0"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(6L), false); - Assertions.assertEquals(2, state.getDataFiles().size(), - "insert-only accepts non-bucket_ data file names"); - Assertions.assertTrue(state.getDeleteDeltas().isEmpty()); - } - - @Test - public void testMissingValidWriteIdsThrows() throws IOException { - createBucketFile("base_0000005", "bucket_00000"); - Map ids = snapshot(6L); - ids.remove(HmsAcidConstants.VALID_WRITEIDS_KEY); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), ids, true)); - Assertions.assertTrue(ex.getMessage().contains("ValidWriteIdList"), ex.getMessage()); - } - - @Test - public void testOriginalFilesWithoutBaseThrows() throws IOException { - // A bare file directly under the partition (no base_) means an unconverted non-ACID table. - Files.write(tempDir.resolve("000000_0"), new byte[] {1}); - - Assertions.assertThrows(UnsupportedOperationException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(6L), true)); - } - - @Test - public void testBaseWithUncommittedVisibilityTxnIsSkipped() throws IOException { - // base_5 was produced by visibility txn 100, which is NOT committed in this snapshot -> it must - // be skipped, falling back to the older committed base_3. This pins the visibility-txn filter. - createBucketFile("base_0000005_v0000100", "bucket_00000"); - createBucketFile("base_0000003", "bucket_00000"); - - // Txn high-watermark 50: visibility txn 100 is not yet valid; base_3's visibility (0) is valid. - ValidTxnList txns = new ValidReadTxnList(new long[0], new BitSet(), 50L, Long.MAX_VALUE); - ValidWriteIdList writeIds = new ValidReaderWriteIdList("db.tbl", new long[0], new BitSet(), 5L); - Map ids = new HashMap<>(); - ids.put(HmsAcidConstants.VALID_TXNS_KEY, txns.writeToString()); - ids.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIds.writeToString()); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), ids, true); - Assertions.assertEquals(1, state.getDataFiles().size()); - Assertions.assertTrue(state.getDataFiles().get(0).location().uri().contains("/base_0000003/"), - "a base whose visibility txn is uncommitted must be skipped for the committed base"); - } - - @Test - public void testHighestBaseInvalidFallsBackToLowerValidBaseAndDropsObsoleteDelta() throws IOException { - // base_8 is beyond the write-id watermark (invalid) and must be rejected despite its higher - // write id, falling back to base_4; delta_2 predates base_4 so it is obsolete and dropped. This - // pins isValidBase discrimination + the selection loop's "delta below the base" rejection. - createBucketFile("base_0000008", "bucket_00000"); - createBucketFile("base_0000004", "bucket_00000"); - createBucketFile("delta_0000002_0000002", "bucket_00000"); - - HiveAcidUtil.AcidState state = HiveAcidUtil.getAcidState( - localFs(), tempDir.toString(), snapshot(5L), true); - - Assertions.assertEquals(1, state.getDataFiles().size(), - "only the best VALID base (base_4) survives; the higher but invalid base_8 is rejected"); - String p = state.getDataFiles().get(0).location().uri(); - Assertions.assertTrue(p.contains("/base_0000004/"), p); - Assertions.assertFalse(p.contains("base_0000008"), "base above the write-id watermark is invalid"); - } - - @Test - public void testNoValidBaseThrowsNotEnoughHistory() throws IOException { - // Only base_8 exists but it is beyond the write-id watermark -> no usable base and no original - // files -> the reader cannot reconstruct the snapshot and must fail loud. - createBucketFile("base_0000008", "bucket_00000"); - - IOException ex = Assertions.assertThrows(IOException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), snapshot(5L), true)); - Assertions.assertTrue(ex.getMessage().contains("Not enough history"), ex.getMessage()); - } - - @Test - public void testMissingValidTxnListThrows() throws IOException { - createBucketFile("base_0000005", "bucket_00000"); - Map ids = snapshot(6L); - ids.remove(HmsAcidConstants.VALID_TXNS_KEY); - - RuntimeException ex = Assertions.assertThrows(RuntimeException.class, - () -> HiveAcidUtil.getAcidState(localFs(), tempDir.toString(), ids, true)); - Assertions.assertTrue(ex.getMessage().contains("ValidTxnList"), ex.getMessage()); - } - - /** - * A Doris {@link LocalFileSystem} that FORBIDS the glob-aware {@link FileSystem#listFiles} / - * {@link FileSystem#listFilesRecursive}. Every test lists through this, so any regression in - * {@link HiveAcidUtil} from the literal {@link FileSystem#list} to {@code listFiles()} fails loud here. - * - *

    The production per-scheme filesystems ({@code DFSFileSystem}, {@code S3CompatibleFileSystem}) override - * {@code listFiles} to treat a location containing {@code [}/{@code *}/{@code ?} as a glob pattern; a real hive - * delta/partition location can contain those, and the old {@code FileSystem.listStatus} never glob-expanded. - * Plain {@link LocalFileSystem#listFiles} is the (literal) interface default, so it alone cannot catch a - * {@code list()->listFiles()} regression — hence this guard. Mirrors {@code FakeFileSystem.listFiles} throwing - * in the non-ACID listing tests. - */ - private static final class LiteralListingLocalFileSystem extends LocalFileSystem { - LiteralListingLocalFileSystem() { - super(Collections.emptyMap()); - } - - @Override - public List listFiles(Location dir) { - throw new AssertionError( - "HiveAcidUtil must list via the literal list(), never the glob-aware listFiles()"); - } - - @Override - public List listFilesRecursive(Location dir) { - throw new AssertionError( - "HiveAcidUtil must list via the literal list(), never the glob-aware listFilesRecursive()"); - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java index 1a654fd327e8f6..d5193e2e68d937 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java @@ -515,7 +515,7 @@ public void scanSkipsOnlyTheFailedPartitionAndPlansTheRest() { lister.error = new HiveDirectoryListingException("dir gone", new IOException("boom")); lister.errorLocation = "loc/dt=1"; HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), + new FakeConnectorContext(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); List ranges = provider.planScan(new FakeSession(), @@ -538,7 +538,7 @@ public void scanFailsLoudOnSystemicFilesystemFailure() { CountingLister lister = new CountingLister(); lister.error = new DorisConnectorException("bad storage config"); HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), + new FakeConnectorContext(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(new FakeSession(), @@ -575,7 +575,7 @@ public void scanProviderServesRepeatedScansFromTheCache() { HiveFileListingCache cache = new HiveFileListingCache(HiveTestProperties.minimal(), lister); // hmsClient is null: with pruned partitions on the handle the scan never calls the metastore. HiveScanPlanProvider provider = new HiveScanPlanProvider( - null, HiveTestProperties.minimal(), new FakeConnectorContext(), new HiveReadTransactionManager(), cache); + null, HiveTestProperties.minimal(), new FakeConnectorContext(), cache); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat") diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java deleted file mode 100644 index 44b2dd58363eb5..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveReadTransactionTest.java +++ /dev/null @@ -1,231 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.connector.hms.HmsClient; -import org.apache.doris.connector.hms.HmsDatabaseInfo; -import org.apache.doris.connector.hms.HmsPartitionInfo; -import org.apache.doris.connector.hms.HmsTableInfo; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Tests the plugin read-side transaction lifecycle ({@link HiveReadTransaction} / - * {@link HiveReadTransactionManager}) against a recording-fake {@link HmsClient}. - * - *

    WHY: reading a transactional Hive table must open a metastore transaction, take a shared read lock - * exactly once (holding it for the whole scan pins a consistent write-id snapshot), and release it by - * committing when the query finishes. Acquiring the lock more than once, or forgetting to commit, either - * corrupts snapshot isolation or leaks the lock. These tests pin that once-per-query lock + commit - * contract.

    - */ -public class HiveReadTransactionTest { - - @Test - public void testBeginOpensTxnAndValidWriteIdsAcquiresLockOnce() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 777L; - client.validIds.put("hive.txn.valid.writeids", "db.tbl:8:x"); - - HiveReadTransaction txn = new HiveReadTransaction( - "q1", "alice", "db", "tbl", true, client); - txn.begin(); - Assertions.assertEquals(List.of("openTxn:alice"), client.calls); - - txn.addPartition("dt=2026-01-01"); - Map ids = txn.getValidWriteIds(); - Assertions.assertEquals("db.tbl:8:x", ids.get("hive.txn.valid.writeids")); - // The lock must be scoped to the transaction, the (db, table), and the added partitions. - Assertions.assertEquals(777L, client.lockTxnId); - Assertions.assertEquals("db.tbl", client.getValidWriteIdsTable); - Assertions.assertEquals(777L, client.getValidWriteIdsTxnId); - Assertions.assertEquals(List.of("dt=2026-01-01"), client.lockPartitions); - - // Second call must be memoized: no extra lock / getValidWriteIds round-trips. - Map ids2 = txn.getValidWriteIds(); - Assertions.assertSame(ids, ids2); - Assertions.assertEquals(1, count(client.calls, "acquireSharedLock:q1")); - Assertions.assertEquals(1, count(client.calls, "getValidWriteIds:db.tbl")); - } - - @Test - public void testCommitCommitsTheTxn() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 42L; - HiveReadTransaction txn = new HiveReadTransaction("q1", "bob", "db", "t", false, client); - txn.begin(); - txn.commit(); - Assertions.assertTrue(client.calls.contains("commitTxn:42"), client.calls.toString()); - } - - @Test - public void testManagerRegisterBeginsAndDeregisterCommits() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 5L; - HiveReadTransactionManager mgr = new HiveReadTransactionManager(); - - HiveReadTransaction txn = new HiveReadTransaction("q1", "u", "db", "t", true, client); - mgr.register(txn); - Assertions.assertEquals(1, count(client.calls, "openTxn:u"), "register opens the txn"); - - mgr.deregister("q1"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:5"), "deregister commits the txn"); - - // Idempotent: a second deregister and an unknown query must be no-ops. - mgr.deregister("q1"); - mgr.deregister("unknown"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:5"), "no double commit"); - } - - @Test - public void testScanProviderReleaseReadTransactionCommitsViaSharedManager() { - RecordingHmsClient client = new RecordingHmsClient(); - client.openTxnReturn = 9L; - // register and the provider's release MUST share the same per-connector manager (HiveConnector injects - // one instance into every provider), so the release finds and commits the txn register opened. - HiveReadTransactionManager mgr = new HiveReadTransactionManager(); - HiveScanPlanProvider provider = new HiveScanPlanProvider(client, HiveTestProperties.minimal(), - new FakeConnectorContext(), mgr, new HiveFileListingCache(HiveTestProperties.minimal())); - - // A transactional-hive scan opened a read txn (as planAcidScan does via mgr.register), taking the shared - // read lock. - mgr.register(new HiveReadTransaction("q9", "u", "db", "t", true, client)); - Assertions.assertEquals(1, count(client.calls, "openTxn:u"), "the scan opened a read txn"); - - // The query-finish callback routes through the provider's releaseReadTransaction, which must commit the - // txn (releasing the shared read lock) exactly once — otherwise the lock leaks for the metastore's life. - provider.releaseReadTransaction("q9"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:9"), - "releaseReadTransaction must commit the txn (release the shared read lock) exactly once"); - - // Idempotent: a second release, or a release for a query that opened no txn, is a safe no-op. - provider.releaseReadTransaction("q9"); - provider.releaseReadTransaction("never-opened"); - Assertions.assertEquals(1, count(client.calls, "commitTxn:9"), "no double commit on repeat release"); - } - - private static int count(List calls, String prefix) { - int n = 0; - for (String c : calls) { - if (c.startsWith(prefix)) { - n++; - } - } - return n; - } - - /** - * Recording {@link HmsClient} fake: stubs the abstract read surface and records the four read-ACID - * primitives the read transaction drives. All other primitives keep their default {@code throw}. - */ - private static final class RecordingHmsClient implements HmsClient { - final List calls = new ArrayList<>(); - long openTxnReturn; - long lockTxnId; - List lockPartitions; - long getValidWriteIdsTxnId; - String getValidWriteIdsTable; - final Map validIds = new HashMap<>(); - - @Override - public long openTxn(String user) { - calls.add("openTxn:" + user); - return openTxnReturn; - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - calls.add("acquireSharedLock:" + queryId + ":" + txnId); - this.lockTxnId = txnId; - this.lockPartitions = new ArrayList<>(partitionNames); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - calls.add("getValidWriteIds:" + fullTableName + ":" + currentTransactionId); - this.getValidWriteIdsTxnId = currentTransactionId; - this.getValidWriteIdsTable = fullTableName; - return validIds; - } - - @Override - public void commitTxn(long txnId) { - calls.add("commitTxn:" + txnId); - } - - // ---- unused abstract read surface ---- - - @Override - public List listDatabases() { - return Collections.emptyList(); - } - - @Override - public HmsDatabaseInfo getDatabase(String dbName) { - throw new UnsupportedOperationException("getDatabase"); - } - - @Override - public List listTables(String dbName) { - return Collections.emptyList(); - } - - @Override - public boolean tableExists(String dbName, String tableName) { - return false; - } - - @Override - public HmsTableInfo getTable(String dbName, String tableName) { - throw new UnsupportedOperationException("getTable"); - } - - @Override - public Map getDefaultColumnValues(String dbName, String tableName) { - return Collections.emptyMap(); - } - - @Override - public List listPartitionNames(String dbName, String tableName, int maxParts) { - return Collections.emptyList(); - } - - @Override - public List getPartitions(String dbName, String tableName, - List partNames) { - return Collections.emptyList(); - } - - @Override - public HmsPartitionInfo getPartition(String dbName, String tableName, List values) { - throw new UnsupportedOperationException("getPartition"); - } - - @Override - public void close() { - } - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java index e4b45ceaa12401..cefeb2b13e985a 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanBatchModeTest.java @@ -37,7 +37,6 @@ import org.apache.doris.filesystem.FileSystem; import org.apache.doris.foundation.format.FileCompression; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TFileScanRangeParams; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -64,9 +63,8 @@ * *

    WHY (Rule 9): the two overrides encode two correctness-critical decisions.

    *
      - *
    • {@code supportsBatchScan}: batch iff PARTITIONED and NOT transactional. ACID tables are excluded so the - * per-batch resolution does not open (and leak) a metastore read transaction per batch — they keep the - * synchronous path.
    • + *
    • {@code supportsBatchScan}: partitioned tables can use batch planning; transactional tables + * are rejected because their read visibility is no longer supported.
    • *
    • {@code planScanForPartitionBatch}: hive's {@code planScan} resolves partitions from the handle's pruned * set and IGNORES the passed partition set, so hive MUST override the batch hook to scope resolution to the * batch. If it inherited the SPI default (which re-runs the whole-pruned-set {@code planScan} per batch), @@ -106,7 +104,7 @@ public String normalizeStorageUri(String uri) { } }; HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), context, - new HiveReadTransactionManager(), cache); + cache); HiveTableHandle handle = parquetHandle(Collections.emptyList()); try (ConnectorSplitSource source = provider.streamSplits(new FakeSession(), ConnectorScanRequest.builder(handle, Collections.emptyList()) @@ -361,7 +359,7 @@ public void streamingDecisionUsesCachedCountWithoutListing() { return Collections.nCopies(3, new HiveFileStatus(location + "/data.parquet", 10, 1)); }); HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), - new FakeConnectorContext(), new HiveReadTransactionManager(), cache); + new FakeConnectorContext(), cache); HiveTableHandle handle = parquetHandle(Collections.emptyList()); FakeSession session = new FakeSession(Collections.singletonMap("num_files_in_batch_mode", "2")); Assertions.assertTrue(provider.streamingSplitEstimate(session, handle, Optional.empty(), false) >= 0); @@ -408,7 +406,7 @@ public void supportsBatchScanIsFalseForUnpartitionedTable() { } @Test - public void supportsBatchScanIsFalseForTransactionalPartitionedTable() { + public void supportsBatchScanRejectsTransactionalPartitionedTable() { HiveScanPlanProvider provider = provider(null, new CountingLister()); Map txnParams = new HashMap<>(); txnParams.put("transactional", "true"); @@ -416,10 +414,8 @@ public void supportsBatchScanIsFalseForTransactionalPartitionedTable() { .partitionKeyNames(PART_KEYS) .tableParameters(txnParams) .build(); - // WHY (Rule 9): ACID tables are excluded DELIBERATELY — running planScanForPartitionBatch per batch on - // background threads would open (and leak) a metastore read transaction per batch. They keep the - // synchronous path even though they are partitioned. - Assertions.assertFalse(provider.supportsBatchScan(new FakeSession(), handle)); + Assertions.assertThrows(DorisConnectorException.class, + () -> provider.supportsBatchScan(new FakeSession(), handle)); } // ==================== planScanForPartitionBatch: scoped to the batch, no duplication ==================== @@ -863,41 +859,6 @@ public void partitionBatchPlanningNeverUsesStatementReuse() { "partition batches must not be planned through the statement reuse memo"); } - @Test - public void transactionalScanNeverUsesStatementReuse() { - // Each transactional scan opens its own read transaction pinning a write-id snapshot; a reused plan would - // skip it. The fake metastore refuses the transaction, so planning stops right after asking for it. - // MUTATION: dropping the transactional bypass in planScan -> the scan installs the statement reuse memo - // before planning -> red. - int[] requestedTransactions = new int[1]; - HmsClient hmsClient = new FakeHmsClient() { - @Override - public long openTxn(String user) { - requestedTransactions[0]++; - throw new UnsupportedOperationException("no read transactions in this test"); - } - }; - HiveScanPlanProvider provider = provider(hmsClient, new CountingLister()); - Map insertOnly = new HashMap<>(); - insertOnly.put("transactional", "true"); - insertOnly.put("transactional_properties", "insert_only"); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .partitionKeyNames(PART_KEYS) - .tableParameters(insertOnly) - .prunedPartitions(Collections.singletonList(part("year=2024/month=01"))) - .build(); - TestStatementScope scope = new TestStatementScope(); - ConnectorSession session = new ScopeSession(7L, "same-statement", scope); - - Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(session, scanOf(handle))); - - Assertions.assertEquals(1, requestedTransactions[0], "a transactional scan asks for its read transaction"); - Assertions.assertFalse(scope.contains(HiveScanPlanProvider.SCAN_REUSE_NAMESPACE + ":7:same-statement"), - "a transactional scan must not be planned through the statement reuse memo"); - } - private static HiveTableHandle textHandle(String tableName, String serde, HmsPartitionInfo partition) { return new HiveTableHandle.Builder("db", tableName, HiveTableType.HIVE) .inputFormat(TEXT_INPUT_FORMAT) @@ -928,7 +889,7 @@ public String normalizeStorageUri(String rawUri) { } }; HiveScanPlanProvider provider = new HiveScanPlanProvider(new FakeHmsClient(), - HiveTestProperties.minimal(), normCtx, new HiveReadTransactionManager(), + HiveTestProperties.minimal(), normCtx, new HiveFileListingCache(HiveTestProperties.minimal(), s3aLister)); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat(PARQUET_INPUT_FORMAT) @@ -961,8 +922,7 @@ public Map getBackendStorageProperties() { }; HiveScanPlanProvider provider = new HiveScanPlanProvider(new FakeHmsClient(), HiveTestProperties.with("s3.access_key", "aliasAK"), credCtx, - new HiveReadTransactionManager(), - new HiveFileListingCache(HiveTestProperties.minimal(), new CountingLister())); + new HiveFileListingCache(HiveTestProperties.minimal(), new CountingLister())); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .inputFormat(PARQUET_INPUT_FORMAT) .serializationLib(PARQUET_SERDE) @@ -1001,74 +961,6 @@ public void getScanNodePropertiesUsesCsvTableParametersForPartitionedAndUnpartit } } - // ============ #65437: scan-level transactional_hive marker (FileScannerV2 exclusion) ============ - // Sibling to supportsBatchScan's ACID exclusion above. BE picks FileScannerV2 from SCAN-LEVEL params before the - // per-range splits arrive, and V2 does not apply ACID delete deltas. A full-ACID hive scan must therefore carry - // the "transactional_hive" marker at scan level (not only per-range) so BE keeps it on FileScanner V1 — - // otherwise deleted rows reappear. - - @Test - public void getScanNodePropertiesEmitsTransactionalHiveForFullAcid() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - Map txnParams = new HashMap<>(); - txnParams.put("transactional", "true"); // full-ACID: transactional AND not insert_only - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .tableParameters(txnParams) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - // WHY (Rule 9): drives the scan-level stamp so BE excludes the full-ACID read from FileScannerV2. - Assertions.assertEquals("true", props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void getScanNodePropertiesOmitsTransactionalHiveForInsertOnly() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - Map params = new HashMap<>(); - params.put("transactional", "true"); - params.put("transactional_properties", "insert_only"); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .tableParameters(params) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - // Insert-only ACID reads have no delete deltas, so FileScannerV2 is correct for them: the marker gates on - // isFullAcid() (NOT isTransactional()), so it must be ABSENT here to keep the V2 fast path. - Assertions.assertNull(props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void getScanNodePropertiesOmitsTransactionalHiveForNonTransactional() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) - .inputFormat(PARQUET_INPUT_FORMAT) - .serializationLib(PARQUET_SERDE) - .build(); - Map props = provider.getScanNodeProperties( - new FakeSession(), handle, Collections.emptyList(), Optional.empty()); - Assertions.assertNull(props.get(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE)); - } - - @Test - public void populateScanLevelParamsStampsTransactionalHiveOnlyWhenSignalled() { - HiveScanPlanProvider provider = provider(new FakeHmsClient(), new CountingLister()); - // With the signal -> scan-level table_format_type = "transactional_hive" (the exact string BE matches on, - // identical to the per-range HiveScanRange marker). - TFileScanRangeParams marked = new TFileScanRangeParams(); - provider.populateScanLevelParams(marked, - Collections.singletonMap(HiveScanPlanProvider.PROP_TRANSACTIONAL_HIVE, "true")); - Assertions.assertTrue(marked.isSetTableFormatParams()); - Assertions.assertEquals("transactional_hive", marked.getTableFormatParams().getTableFormatType()); - // Without the signal -> untouched, so non-ACID hive keeps the FileScannerV2 fast path. - TFileScanRangeParams unmarked = new TFileScanRangeParams(); - provider.populateScanLevelParams(unmarked, Collections.emptyMap()); - Assertions.assertFalse(unmarked.isSetTableFormatParams()); - } - // ===== helpers ===== @Test @@ -1101,7 +993,7 @@ public void usesHiveParquetInt96TimeZone() { private static HiveScanPlanProvider provider( HmsClient hmsClient, HiveFileListingCache.DirectoryLister lister) { return new HiveScanPlanProvider(hmsClient, HiveTestProperties.minimal(), new FakeConnectorContext(), - new HiveReadTransactionManager(), new HiveFileListingCache(HiveTestProperties.minimal(), lister)); + new HiveFileListingCache(HiveTestProperties.minimal(), lister)); } private static HmsPartitionInfo part(String name) { diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java deleted file mode 100644 index 5ba592feb1967e..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangeAcidTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.apache.doris.thrift.TFileRangeDesc; -import org.apache.doris.thrift.TTableFormatFileDesc; -import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; -import org.apache.doris.thrift.TTransactionalHiveDesc; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.List; - -/** - * Tests that {@link HiveScanRange} carries ACID delete-delta info to the BE. - * - *

      WHY: for transactional Hive tables the BE applies row-level deletes by reading the - * delete-delta files listed per partition. The delta descriptor needs BOTH the directory - * AND the file names within it. If the file names are dropped, the BE cannot locate the - * delete records and silently under-deletes — returning rows that were logically deleted. - * These tests pin the "dir|file1,file2" encode/decode round-trip end to end.

      - */ -public class HiveScanRangeAcidTest { - - @Test - public void testDeleteDeltaCarriesDirectoryAndFileNames() { - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/delta_0000005_0000005/bucket_00000") - .acidInfo("/tbl/p=1", Arrays.asList( - "/tbl/p=1/delete_delta_0000003_0000003|bucket_00000,bucket_00001", - "/tbl/p=1/delete_delta_0000004_0000004|bucket_00000")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - Assertions.assertTrue(formatDesc.isSetTransactionalHiveParams(), - "transactional_hive range must emit transactional params"); - TTransactionalHiveDesc txnDesc = formatDesc.getTransactionalHiveParams(); - Assertions.assertEquals("/tbl/p=1", txnDesc.getPartition()); - - List deltas = txnDesc.getDeleteDeltas(); - Assertions.assertEquals(2, deltas.size()); - - TTransactionalHiveDeleteDeltaDesc first = deltas.get(0); - Assertions.assertEquals("/tbl/p=1/delete_delta_0000003_0000003", - first.getDirectoryLocation()); - // The regression: file names must survive the "dir|file1,file2" round-trip. - Assertions.assertEquals(Arrays.asList("bucket_00000", "bucket_00001"), - first.getFileNames()); - - TTransactionalHiveDeleteDeltaDesc second = deltas.get(1); - Assertions.assertEquals("/tbl/p=1/delete_delta_0000004_0000004", - second.getDirectoryLocation()); - Assertions.assertEquals(Arrays.asList("bucket_00000"), second.getFileNames()); - } - - @Test - public void testDeleteDeltaWithoutFileNamesLeavesFileNamesUnset() { - // A directory-only encoding (no '|') must still set the directory and simply - // carry no file names, rather than mis-parsing the whole string as a directory - // with a bogus trailing file name. - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/base_0000005/bucket_00000") - .acidInfo("/tbl/p=1", Arrays.asList("/tbl/p=1/delete_delta_dir_only")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - TTransactionalHiveDeleteDeltaDesc delta = - formatDesc.getTransactionalHiveParams().getDeleteDeltas().get(0); - Assertions.assertEquals("/tbl/p=1/delete_delta_dir_only", delta.getDirectoryLocation()); - Assertions.assertFalse(delta.isSetFileNames()); - } - - @Test - public void testNonTransactionalRangeEmitsNoTransactionalParams() { - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/000000_0") - .fileFormat("parquet") - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - range.populateRangeParams(formatDesc, new TFileRangeDesc()); - - Assertions.assertFalse(formatDesc.isSetTransactionalHiveParams()); - } -} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java index 490b4daf980b11..13cfd67ac1d401 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanRangePartitionValuesTest.java @@ -122,25 +122,4 @@ public void testUnpartitionedEmitsNoColumnsFromPath() { Assertions.assertFalse(rangeDesc.isSetColumnsFromPathIsNull()); } - @Test - public void testAcidTableEmitsBothTransactionalParamsAndPartitionValues() { - // ACID (transactional_hive) ranges must still carry columns-from-path in addition to the - // transactional delete-delta params; the partition-value rebuild runs regardless of ACID. - HiveScanRange range = HiveScanRange.builder() - .path("/tbl/dt=2024-01-01/delta_0000005_0000005/bucket_00000") - .partitionValues(orderedPartitionValues("dt", "2024-01-01")) - .acidInfo("/tbl/dt=2024-01-01", Arrays.asList( - "/tbl/dt=2024-01-01/delete_delta_0000003_0000003|bucket_00000")) - .build(); - - TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); - TFileRangeDesc rangeDesc = new TFileRangeDesc(); - range.populateRangeParams(formatDesc, rangeDesc); - - Assertions.assertTrue(formatDesc.isSetTransactionalHiveParams(), - "ACID range must still emit transactional params"); - Assertions.assertEquals(Collections.singletonList("dt"), rangeDesc.getColumnsFromPathKeys()); - Assertions.assertEquals(Collections.singletonList("2024-01-01"), rangeDesc.getColumnsFromPath()); - Assertions.assertEquals(Collections.singletonList(false), rangeDesc.getColumnsFromPathIsNull()); - } } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java new file mode 100644 index 00000000000000..b98c9dac162f7f --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveUnsupportedReadsTest.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +public class HiveUnsupportedReadsTest { + @Test + public void rejectTransactionalTablesBeforePlanningOrCountPushdown() { + HiveScanPlanProvider provider = new HiveScanPlanProvider(null, HiveTestProperties.minimal(), + null, null); + for (String mode : new String[] {"default", "insert_only"}) { + HiveTableHandle handle = new HiveTableHandle.Builder("db", "tbl", HiveTableType.HIVE) + .inputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat") + .tableParameters(Map.of("transactional", "true", "transactional_properties", mode)) + .build(); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.getScanNodeProperties(new ScopeSession(1L, "q", + new TestStatementScope()), handle, Collections.emptyList(), Optional.empty())); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, Collections.emptyList()).build(); + Assertions.assertThrows(DorisConnectorException.class, () -> provider.planScan(null, request)); + Assertions.assertThrows(DorisConnectorException.class, () -> provider.streamSplits(null, request)); + Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planScanForPartitionBatch(null, request, Collections.singletonList("p=1"))); + Assertions.assertTrue(error.getMessage().contains( + "Hive transactional table reads are no longer supported")); + } + } +} diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index f9125dfe66b190..37b5aae4727859 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -604,27 +604,6 @@ public boolean partitionExists(String dbName, String tableName, List par return delegate.partitionExists(dbName, tableName, partitionValues); } - @Override - public long openTxn(String user) { - return delegate.openTxn(user); - } - - @Override - public void commitTxn(long txnId) { - delegate.commitTxn(txnId); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - return delegate.getValidWriteIds(fullTableName, currentTransactionId); - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - delegate.acquireSharedLock(queryId, txnId, user, dbName, tableName, partitionNames, timeoutMs); - } - @Override public long acquireExclusiveTableLock(String queryId, String user, String dbName, String tableName, long timeoutMs) { diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java deleted file mode 100644 index eac80de20042a3..00000000000000 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsAcidConstants.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hms; - -/** - * Hadoop-configuration keys that carry the ACID snapshot (valid transaction list + valid write-id - * list) from FE to BE for transactional Hive reads. - * - *

      These are a fixed Hive/BE contract: the same literal strings fe-core's {@code AcidUtil} - * produces (via {@link HmsClient#getValidWriteIds}) and consumes. Defined once here so the write-id - * producer ({@link ThriftHmsClient}) and the future read-side ACID descent share one source of - * truth.

      - */ -public final class HmsAcidConstants { - - /** Serialized {@code ValidTxnList}. */ - public static final String VALID_TXNS_KEY = "hive.txn.valid.txns"; - - /** Serialized {@code ValidWriteIdList}. */ - public static final String VALID_WRITEIDS_KEY = "hive.txn.valid.writeids"; - - private HmsAcidConstants() { - } -} diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java index ff81a8cfee9be2..1ce842d1a0cbb9 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java @@ -369,66 +369,6 @@ default boolean partitionExists(String dbName, String tableName, throw new UnsupportedOperationException("partitionExists is not supported by this client"); } - // ========== Phase 4: ACID transactions ========== - // - // Read-side ACID primitives for transactional Hive tables. Like the Phase 3 block, they default - // to throwing so read-only / non-transactional clients need not implement them. - - /** - * Open a Hive ACID transaction. - * - * @param user the user opening the transaction - * @return the new transaction id - * @throws HmsClientException if the operation fails - */ - default long openTxn(String user) { - throw new UnsupportedOperationException("openTxn is not supported by this client"); - } - - /** - * Commit a Hive ACID transaction (also releases the transaction's locks). - * - * @param txnId the transaction id - * @throws HmsClientException if the operation fails - */ - default void commitTxn(long txnId) { - throw new UnsupportedOperationException("commitTxn is not supported by this client"); - } - - /** - * Get the valid-transaction / valid-write-id snapshot for a transactional table, as the two - * Hadoop-configuration entries ({@link HmsAcidConstants}) that BE consumes for ACID reads. On a - * metastore-incompatibility error the implementation degrades to a max watermark rather than - * failing the read. - * - * @param fullTableName fully-qualified "db.table" - * @param currentTransactionId the reader's open transaction id - * @return map with {@link HmsAcidConstants#VALID_TXNS_KEY} and - * {@link HmsAcidConstants#VALID_WRITEIDS_KEY} - * @throws HmsClientException if the operation fails - */ - default Map getValidWriteIds(String fullTableName, long currentTransactionId) { - throw new UnsupportedOperationException("getValidWriteIds is not supported by this client"); - } - - /** - * Acquire a shared (read) lock over a table and, if given, specific partitions, polling until the - * lock is granted or the timeout elapses. - * - * @param queryId the query id (lock owner) - * @param txnId the transaction id - * @param user the requesting user - * @param dbName database name - * @param tableName table name - * @param partitionNames partitions to lock; empty locks the whole table - * @param timeoutMs maximum time to wait for the lock, in milliseconds - * @throws HmsClientException if the lock cannot be acquired within the timeout - */ - default void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - throw new UnsupportedOperationException("acquireSharedLock is not supported by this client"); - } - /** * Acquire a standalone exclusive lock over a non-transactional Hive table. * diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java index 9efde69b5a12d1..d9b388d98df0e2 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java @@ -25,11 +25,6 @@ import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -46,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; @@ -60,15 +54,12 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import shade.doris.hive.org.apache.thrift.TApplicationException; import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -532,85 +523,6 @@ public boolean partitionExists(String dbName, String tableName, }); } - // ========== Phase 4: ACID transactions ========== - - @Override - public long openTxn(String user) { - return execute(client -> client.openTxn(user)); - } - - @Override - public void commitTxn(long txnId) { - execute(client -> { - client.commitTxn(txnId); - return null; - }); - } - - @Override - public Map getValidWriteIds(String fullTableName, long currentTransactionId) { - Map conf = new HashMap<>(); - try { - return execute(client -> { - // Use the recent snapshot of valid transactions (no currentTxn arg): passing - // currentTransactionId here would break Hive's delta-directory listing if a major - // compaction removed deltas for transactions that were valid when this txn opened. - ValidTxnList validTransactions = client.getValidTxns(); - List tableValidWriteIdsList = client.getValidWriteIds( - Collections.singletonList(fullTableName), validTransactions.toString()); - if (tableValidWriteIdsList.size() != 1) { - throw new HmsClientException("tableValidWriteIdsList's size should be 1"); - } - ValidTxnWriteIdList validTxnWriteIdList = TxnUtils.createValidTxnWriteIdList( - currentTransactionId, tableValidWriteIdsList); - ValidWriteIdList writeIdList = - validTxnWriteIdList.getTableValidWriteIdList(fullTableName); - conf.put(HmsAcidConstants.VALID_TXNS_KEY, validTransactions.writeToString()); - conf.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIdList.writeToString()); - return conf; - }); - } catch (Exception e) { - // Older / incompatible metastores may lack these APIs; degrade to a max watermark - // instead of failing the scan (mirrors legacy ThriftHMSCachedClient). - LOG.warn("failed to get valid write ids for {}, transaction {}", - fullTableName, currentTransactionId, e); - ValidTxnList validTransactions = new ValidReadTxnList( - new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); - ValidWriteIdList writeIdList = new ValidReaderWriteIdList( - fullTableName, new long[0], new BitSet(), Long.MAX_VALUE); - conf.put(HmsAcidConstants.VALID_TXNS_KEY, validTransactions.writeToString()); - conf.put(HmsAcidConstants.VALID_WRITEIDS_KEY, writeIdList.writeToString()); - return conf; - } - } - - @Override - public void acquireSharedLock(String queryId, long txnId, String user, String dbName, - String tableName, List partitionNames, long timeoutMs) { - LockRequestBuilder requestBuilder = new LockRequestBuilder(queryId) - .setTransactionId(txnId) - .setUser(user); - for (LockComponent component - : createLockComponentsForRead(dbName, tableName, partitionNames)) { - requestBuilder.addLockComponent(component); - } - LockRequest lockRequest = requestBuilder.build(); - LockResponse response = execute(client -> client.lock(lockRequest)); - long start = System.currentTimeMillis(); - while (response.getState() == LockState.WAITING) { - if (System.currentTimeMillis() - start > timeoutMs) { - throw new HmsClientException("acquire lock timeout for txn " + txnId - + " of query " + queryId + ", timeout(ms): " + timeoutMs); - } - long lockId = response.getLockid(); - response = execute(client -> client.checkLock(lockId)); - } - if (response.getState() != LockState.ACQUIRED) { - throw new HmsClientException( - "failed to acquire lock, lock in state " + response.getState()); - } - } - @Override public long acquireExclusiveTableLock(String queryId, String user, String dbName, String tableName, long timeoutMs) { @@ -700,34 +612,6 @@ public List getNextNotification(long lastEventId, int maxE }); } - private static List createLockComponentsForRead(String dbName, String tableName, - List partitionNames) { - List components = new ArrayList<>( - partitionNames.isEmpty() ? 1 : partitionNames.size()); - if (partitionNames.isEmpty()) { - components.add(createLockComponentForRead(dbName, tableName, null)); - } else { - for (String partitionName : partitionNames) { - components.add(createLockComponentForRead(dbName, tableName, partitionName)); - } - } - return components; - } - - private static LockComponent createLockComponentForRead(String dbName, String tableName, - String partitionName) { - LockComponentBuilder builder = new LockComponentBuilder(); - builder.setShared(); - builder.setOperationType(DataOperationType.SELECT); - builder.setDbName(dbName); - builder.setTableName(tableName); - if (partitionName != null) { - builder.setPartitionName(partitionName); - } - builder.setIsTransactional(true); - return builder.build(); - } - @Override public synchronized void close() throws IOException { if (closed) { diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java index 6ca79cb1beabd0..a3e6cfdbe308f1 100644 --- a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java @@ -18,7 +18,6 @@ package org.apache.doris.connector.hms; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.LockComponent; @@ -29,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import shade.doris.hive.org.apache.thrift.TApplicationException; @@ -38,10 +36,8 @@ import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -65,29 +61,6 @@ */ public class ThriftHmsClientWriteAcidTest { - // ---- openTxn / commitTxn -------------------------------------------------------------------- - - @Test - public void testOpenTxnForwardsUserAndReturnsId() { - RecordingClient fake = new RecordingClient().stub("openTxn", 42L); - ThriftHmsClient client = newClient(fake); - - long txnId = client.openTxn("alice"); - - Assertions.assertEquals(42L, txnId); - Assertions.assertEquals("alice", argsOf(fake, "openTxn")[0]); - } - - @Test - public void testCommitTxnForwardsId() { - RecordingClient fake = new RecordingClient(); - ThriftHmsClient client = newClient(fake); - - client.commitTxn(7L); - - Assertions.assertEquals(7L, argsOf(fake, "commitTxn")[0]); - } - // ---- dropPartition / partitionExists -------------------------------------------------------- @Test @@ -422,116 +395,6 @@ public void testUpdatePartitionStatisticsRejectsAmbiguousName() { current -> HmsPartitionStatistics.EMPTY)); } - // ---- getValidWriteIds ----------------------------------------------------------------------- - - @Test - public void testGetValidWriteIdsSuccessPathEmitsSnapshotThenWriteIds() { - // Primary read-ACID path: a compatible metastore returns exactly one write-id list. A - // distinctive snapshot (high-watermark 100, NOT the fallback's Long.MAX_VALUE) makes this test - // fail if the code silently degrades to the fallback branch. - ValidReadTxnList snapshot = new ValidReadTxnList(new long[0], new BitSet(), 100L, 5L); - TableValidWriteIds writeIds = new TableValidWriteIds( - "db.t", 100L, Collections.emptyList(), ByteBuffer.allocate(0)); - RecordingClient fake = new RecordingClient() - .stub("getValidTxns", snapshot) - .stub("getValidWriteIds", Collections.singletonList(writeIds)); - ThriftHmsClient client = newClient(fake); - - Map conf = client.getValidWriteIds("db.t", 42L); - - // (a) The recent snapshot string — NOT currentTransactionId (42) — drives the write-id lookup. - Object[] gvwiArgs = argsOf(fake, "getValidWriteIds"); - Assertions.assertEquals(Collections.singletonList("db.t"), gvwiArgs[0]); - Assertions.assertEquals(snapshot.toString(), gvwiArgs[1]); - // (b) VALID_TXNS_KEY carries the txn snapshot; VALID_WRITEIDS_KEY carries the write-id list — - // no key swap. Both are the SUCCESS values (snapshot watermark 100), not the fallback watermark. - Assertions.assertEquals(snapshot.writeToString(), conf.get(HmsAcidConstants.VALID_TXNS_KEY)); - Assertions.assertTrue(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY).contains("db.t")); - Assertions.assertNotEquals(conf.get(HmsAcidConstants.VALID_TXNS_KEY), - conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - } - - @Test - public void testGetValidWriteIdsFallsBackToMaxWatermark() { - // An incompatible metastore returns an unexpected write-id list; rather than fail the scan, - // getValidWriteIds must degrade to a max watermark and still emit both config keys. - ValidReadTxnList snapshot = new ValidReadTxnList(); - RecordingClient fake = new RecordingClient() - .stub("getValidTxns", snapshot) - .stub("getValidWriteIds", new ArrayList<>()); // size 0 -> triggers fallback - ThriftHmsClient client = newClient(fake); - - Map conf = client.getValidWriteIds("db.t", 5L); - - // The recent snapshot (not currentTransactionId) was still forwarded before the size guard threw. - Object[] gvwiArgs = argsOf(fake, "getValidWriteIds"); - Assertions.assertEquals(Collections.singletonList("db.t"), gvwiArgs[0]); - Assertions.assertEquals(snapshot.toString(), gvwiArgs[1]); - // Both BE-contract keys are present, and the fallback write-id list is scoped to the table. - Assertions.assertNotNull(conf.get(HmsAcidConstants.VALID_TXNS_KEY)); - Assertions.assertNotNull(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY)); - Assertions.assertTrue(conf.get(HmsAcidConstants.VALID_WRITEIDS_KEY).contains("db.t")); - } - - // ---- acquireSharedLock ---------------------------------------------------------------------- - - @Test - public void testAcquireSharedLockBuildsSharedReadComponentsAndReturnsWhenAcquired() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(1L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q1", 5L, "alice", "db", "t", - Collections.emptyList(), 1000L); - - LockRequest request = (LockRequest) argsOf(fake, "lock")[0]; - Assertions.assertEquals(1, request.getComponent().size()); - LockComponent component = request.getComponent().get(0); - Assertions.assertEquals("db", component.getDbname()); - Assertions.assertEquals("t", component.getTablename()); - Assertions.assertEquals(LockType.SHARED_READ, component.getType()); - Assertions.assertEquals(DataOperationType.SELECT, component.getOperationType()); - } - - @Test - public void testAcquireSharedLockOneComponentPerPartition() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(1L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q1", 5L, "alice", "db", "t", - java.util.Arrays.asList("dt=1", "dt=2"), 1000L); - - LockRequest request = (LockRequest) argsOf(fake, "lock")[0]; - Assertions.assertEquals(2, request.getComponent().size()); - Assertions.assertEquals("dt=1", request.getComponent().get(0).getPartitionname()); - Assertions.assertEquals("dt=2", request.getComponent().get(1).getPartitionname()); - } - - @Test - public void testAcquireSharedLockPollsUntilAcquired() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(2L, LockState.WAITING)) - .stub("checkLock", new LockResponse(2L, LockState.ACQUIRED)); - ThriftHmsClient client = newClient(fake); - - client.acquireSharedLock("q", 5L, "u", "db", "t", Collections.emptyList(), 5000L); - - Assertions.assertTrue(fake.methodNames.contains("checkLock")); - } - - @Test - public void testAcquireSharedLockTimesOut() { - RecordingClient fake = new RecordingClient() - .stub("lock", new LockResponse(3L, LockState.WAITING)) - .stub("checkLock", new LockResponse(3L, LockState.WAITING)); - ThriftHmsClient client = newClient(fake); - - // timeoutMs = -1 => the elapsed guard trips on the first poll iteration, deterministically. - Assertions.assertThrows(HmsClientException.class, () -> - client.acquireSharedLock("q", 5L, "u", "db", "t", Collections.emptyList(), -1L)); - } - @Test public void testExclusiveTableLockIsStandaloneAndExplicitlyReleased() { RecordingClient fake = new RecordingClient() diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java index 42c947c5506776..022ce8c8002e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java @@ -27,7 +27,6 @@ public class FileFormatConstants { public static final String FORMAT_PARQUET = "parquet"; public static final String FORMAT_ORC = "orc"; public static final String FORMAT_JSON = "json"; - public static final String FORMAT_AVRO = "avro"; public static final String FORMAT_WAL = "wal"; public static final String FORMAT_ARROW = "arrow"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java deleted file mode 100644 index 7622cd19644dfb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatProperties.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.fileformat; - -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.thrift.TFileAttributes; -import org.apache.doris.thrift.TFileFormatType; -import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.doris.thrift.TResultFileSinkOptions; - -import java.util.Map; - -public class AvroFileFormatProperties extends FileFormatProperties { - public AvroFileFormatProperties() { - super(TFileFormatType.FORMAT_AVRO, FileFormatProperties.FORMAT_AVRO); - } - - @Override - public void analyzeFileFormatProperties(Map formatProperties, boolean isRemoveOriginProperty) - throws AnalysisException { - } - - @Override - public void fullTResultFileSinkOptions(TResultFileSinkOptions sinkOptions) { - } - - @Override - public TFileAttributes toTFileAttributes() { - TFileAttributes fileAttributes = new TFileAttributes(); - TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); - fileAttributes.setTextParams(fileTextScanRangeParams); - return fileAttributes; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index c6b2a8e4109691..cb3cbe0e0bde7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -96,7 +96,7 @@ public static FileFormatProperties createFileFormatProperties(String formatStrin case FORMAT_JSON: return new JsonFileFormatProperties(); case FORMAT_AVRO: - return new AvroFileFormatProperties(); + throw new AnalysisException("Avro file reads are no longer supported"); case FORMAT_WAL: return new WalFileFormatProperties(); case FORMAT_ARROW: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java index 1608a896b99968..88304a130baeb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java @@ -266,7 +266,7 @@ public void createScanRangeLocationsUnsplittable(FileLoadScanNode.ParamCreateCon context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus, fileStatus.size, columnsFromPath.getValues(), columnsFromPathKeys, columnsFromPath.getIsNull()); @@ -313,7 +313,7 @@ public void createScanRangeLocationsSplittable(FileLoadScanNode.ParamCreateConte context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); // Assign scan range locations only for broker load. // stream load has only one file, and no need to set multi scan ranges. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java index 0b6e73ec4f6b34..8b523b240420d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java @@ -66,41 +66,17 @@ public List getIsNull() { */ public static List parseColumnsFromPath(String filePath, List columnsFromPath) throws UserException { - return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, true, false).getValues(); - } - - /** - * Parses partition column values from a Hive-style file path. - * - * @param filePath absolute file path that contains partition segments - * @param columnsFromPath ordered list of partition column names to extract - * @param caseSensitive whether column name matching is case-sensitive - * @param isACID whether the path follows the ACID layout - * ({@code table/par=val/delta_xxx/file}), which adds one extra path level - * @return ordered list of partition values (parallel to {@code columnsFromPath}) - * @throws UserException if the path does not contain the expected partition segments - */ - public static List parseColumnsFromPath( - String filePath, - List columnsFromPath, - boolean caseSensitive, - boolean isACID) - throws UserException { - return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, caseSensitive, isACID) - .getValues(); + return parseColumnsFromPathWithNullInfo(filePath, columnsFromPath, true).getValues(); } public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( String filePath, List columnsFromPath, - boolean caseSensitive, - boolean isACID) + boolean caseSensitive) throws UserException { if (columnsFromPath == null || columnsFromPath.isEmpty()) { return new ParsedColumnsFromPath(Collections.emptyList(), Collections.emptyList()); } - // ACID paths have one extra level: table/par=val/delta_xxx/file → pathCount = 3 - int pathCount = isACID ? 3 : 2; List expectedColumns = columnsFromPath; if (!caseSensitive) { expectedColumns = new ArrayList<>(columnsFromPath.size()); @@ -117,13 +93,13 @@ public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( Boolean[] columnValueIsNull = new Boolean[expectedColumns.size()]; int size = 0; boolean skipOnce = true; - for (int i = strings.length - pathCount; i >= 0; i--) { + for (int i = strings.length - 2; i >= 0; i--) { String str = strings[i]; if (str != null && str.isEmpty()) { continue; } if (str == null || !str.contains("=")) { - if (!isACID && skipOnce) { + if (skipOnce) { skipOnce = false; continue; } @@ -164,11 +140,10 @@ public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfo( public static ParsedColumnsFromPath parseColumnsFromPathWithNullInfoForLoad( String filePath, List columnsFromPath, - boolean caseSensitive, - boolean isACID) + boolean caseSensitive) throws UserException { ParsedColumnsFromPath parsed = parseColumnsFromPathWithNullInfo( - filePath, columnsFromPath, caseSensitive, isACID); + filePath, columnsFromPath, caseSensitive); List values = new ArrayList<>(parsed.getValues()); List isNull = new ArrayList<>(parsed.getIsNull()); for (int i = 0; i < values.size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java index b1d0d4f480e832..921b88077f3929 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileQueryScanNode.java @@ -505,14 +505,11 @@ private TScanRangeLocations splitToScanRange( FileSplit fileSplit = (FileSplit) split; TScanRangeLocations curLocations = newLocations(); // If fileSplit has partition values, use the values collected from hive partitions. - // Otherwise, use the values in file path. Migrated tables emit PluginDrivenSplit (never a HiveSplit) - // and always carry connector-supplied partition values, so isACID stays false: the connector owns - // ACID delta-dir handling and rewrites columns-from-path in its ConnectorScanRange.populateRangeParams. - boolean isACID = false; + // Otherwise, parse partition values from the file path. FilePartitionUtils.ParsedColumnsFromPath partitionValuesFromPath = fileSplit.getPartitionValues() == null ? FilePartitionUtils.parseColumnsFromPathWithNullInfo( - fileSplit.getPathString(), pathPartitionKeys, false, isACID) + fileSplit.getPathString(), pathPartitionKeys, false) : FilePartitionUtils.normalizeColumnsFromPath(fileSplit.getPartitionValues()); TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValuesFromPath.getValues(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java index 841965f63aac6a..45bc68f13b4ba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java @@ -277,7 +277,7 @@ public void createScanRangeLocationsUnsplittable(NereidsParamCreateContext conte context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus, fileStatus.size, columnsFromPath.getValues(), columnsFromPathKeys, columnsFromPath.getIsNull()); @@ -330,7 +330,7 @@ public void createScanRangeLocationsSplittable(NereidsParamCreateContext context context.params.setCompressType(compressType); FilePartitionUtils.ParsedColumnsFromPath columnsFromPath = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path, - context.fileGroup.getColumnNamesFromPath(), true, false); + context.fileGroup.getColumnNamesFromPath(), true); List columnsFromPathKeys = context.fileGroup.getColumnNamesFromPath(); // Assign scan range locations only for broker load. // stream load has only one file, and no need to set multi scan ranges. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 637fdf5793c226..ef66650ab5ed23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -839,8 +839,6 @@ public String toString() { public static final String EXCHANGE_MULTI_BLOCKS_BYTE_SIZE = "exchange_multi_blocks_byte_size"; - public static final String SKIP_CHECKING_ACID_VERSION_FILE = "skip_checking_acid_version_file"; - public static final String ENABLE_EXTENDED_REGEX = "enable_extended_regex"; public static final String ENABLE_HYPERSCAN_FALLBACK = "enable_hyperscan_fallback"; @@ -3519,11 +3517,6 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { + "non-utf8 characters will be displayed as garbled characters.") public boolean enableTextValidateUtf8 = true; - @VarAttrDef.VarAttr(name = SKIP_CHECKING_ACID_VERSION_FILE, needForward = true, affectQueryResultInPlan = true, - description = "Skip checking transactional hive version file '_orc_acid_version.'" - ) - public boolean skipCheckingAcidVersionFile = false; - @VarAttrDef.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward = true, checker = "checkSqlConvertorFeatures", description = "enable SQL convertor features. Multiple features are separated by commas") diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 629378a73534d6..ac3a58e09adad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -157,6 +157,8 @@ public class VariableMgr { "enable_phrase_query_sequential_opt", "enable_rust_lance_reader", "enable_paimon_cpp_reader", + // Keep old connection-init SET statements valid after removing Hive ACID reads. + "skip_checking_acid_version_file", "enable_nereids_trace", "nereids_trace_event_mode", "shuffled_agg_node_ids", diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java index c473e174189975..7053d256e74296 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java @@ -41,9 +41,8 @@ public HttpStreamTableValuedFunction(Map properties) throws Anal super.parseCommonProperties(properties); if (fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_PARQUET - || fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_AVRO || fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_ORC) { - throw new AnalysisException("http_stream does not yet support parquet, avro and orc"); + throw new AnalysisException("http_stream does not yet support parquet and orc"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java index bbe5e4340ed665..8ad8df14d68e95 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java @@ -133,35 +133,29 @@ public void parseColumnsFromPathPreservesNullMetadataAndInputKeys() throws Excep List partitionKeys = Lists.newArrayList("Region", "Dt"); FilePartitionUtils.ParsedColumnsFromPath parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( "hdfs://host/table/REGION=cn/dt=" + ConnectorPartitionValues.NULL_PARTITION_NAME + "/data.parquet", - partitionKeys, false, false); + partitionKeys, false); Assertions.assertEquals(Lists.newArrayList("cn", ""), parsed.getValues()); Assertions.assertEquals(Lists.newArrayList(false, true), parsed.getIsNull()); Assertions.assertEquals(Lists.newArrayList("Region", "Dt"), partitionKeys); // A literal \N is data on the query path. parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( - "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true, false); + "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true); Assertions.assertEquals(Collections.singletonList("\\N"), parsed.getValues()); Assertions.assertEquals(Collections.singletonList(false), parsed.getIsNull()); - - // An ACID layout has one more directory level between the partition and the file. - parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfo( - "hdfs://host/table/p=value/delta_1_1/bucket_00000", Collections.singletonList("p"), true, true); - Assertions.assertEquals(Collections.singletonList("value"), parsed.getValues()); - Assertions.assertEquals(Collections.singletonList(false), parsed.getIsNull()); } @Test public void parseColumnsFromPathForLoadKeepsLegacyNullSemantics() throws Exception { FilePartitionUtils.ParsedColumnsFromPath parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad( - "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true, false); + "hdfs://host/table/p=\\N/data.orc", Collections.singletonList("p"), true); Assertions.assertEquals(Collections.singletonList(""), parsed.getValues()); Assertions.assertEquals(Collections.singletonList(true), parsed.getIsNull()); // Other values, including the Hive default partition, parse as on the query path. parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad( "hdfs://host/table/p=a/q=" + ConnectorPartitionValues.NULL_PARTITION_NAME + "/data.orc", - Lists.newArrayList("p", "q"), true, false); + Lists.newArrayList("p", "q"), true); Assertions.assertEquals(Lists.newArrayList("a", ""), parsed.getValues()); Assertions.assertEquals(Lists.newArrayList(false, true), parsed.getIsNull()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java new file mode 100644 index 00000000000000..61b7b9947247b9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/UnsupportedExternalReadsTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; + +public class UnsupportedExternalReadsTest { + @Test + public void testAvroFileFormatIsRejected() { + for (String format : new String[] {"avro", "AVRO"}) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> FileFormatProperties.createFileFormatProperties(format)); + Assertions.assertTrue(error.getMessage().contains("Avro file reads are no longer supported")); + } + } + + @Test + public void testLakeSoulCatalogIsRejected() { + CatalogLog log = new CatalogLog(); + log.setCatalogId(1L); + log.setCatalogName("removed_catalog"); + log.setProps(new HashMap<>(Collections.singletonMap("type", "lakesoul"))); + DdlException error = Assertions.assertThrows(DdlException.class, () -> CatalogFactory.createFromLog(log)); + Assertions.assertTrue(error.getMessage().contains("Lakesoul catalog is no longer supported")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java deleted file mode 100644 index 03323d8e159432..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/fileformat/AvroFileFormatPropertiesTest.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.fileformat; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - - -public class AvroFileFormatPropertiesTest { - - private AvroFileFormatProperties avroFileFormatProperties; - - @BeforeEach - public void setUp() { - avroFileFormatProperties = new AvroFileFormatProperties(); - } - - @Test - public void testAnalyzeFileFormatProperties() { - Map properties = new HashMap<>(); - // Add properties if needed - avroFileFormatProperties.analyzeFileFormatProperties(properties, true); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java index 0be56ce2953520..1bc0e9a004b605 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeHiveStreamingTest.java @@ -28,7 +28,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.connector.hive.HiveCatalogProperties; import org.apache.doris.connector.hive.HiveFileListingCache; -import org.apache.doris.connector.hive.HiveReadTransactionManager; import org.apache.doris.connector.hive.HiveScanPlanProvider; import org.apache.doris.connector.hive.HiveScanRange; import org.apache.doris.connector.hive.HiveTableHandle; @@ -227,7 +226,7 @@ public void close() { HiveCatalogProperties properties = HiveCatalogProperties.of(Collections.singletonMap( HiveCatalogProperties.HIVE_METASTORE_URIS, "thrift://localhost:9083")); HiveScanPlanProvider provider = new HiveScanPlanProvider(hms, properties, context, - new HiveReadTransactionManager(), new HiveFileListingCache(properties)); + new HiveFileListingCache(properties)); HiveTableHandle handle = new HiveTableHandle.Builder("db", "t", HiveTableType.HIVE) .partitionKeyNames(Collections.singletonList("pt")).location("s3://bucket/") .inputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat") diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java new file mode 100644 index 00000000000000..507bfbeda68471 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/RemovedAcidSessionVariableTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.analysis.SetType; +import org.apache.doris.analysis.SetVar; +import org.apache.doris.analysis.VariableExpr; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.trees.expressions.literal.Literal; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +class RemovedAcidSessionVariableTest { + @Test + void removedAcidVariableRemainsANoOpForSessionInitialization() throws Exception { + SessionVariable variables = new SessionVariable(); + for (String name : new String[] {"skip_checking_acid_version_file", + "skip_checking_acid_version_file".toUpperCase(Locale.ROOT)}) { + for (boolean value : new boolean[] {true, false}) { + SetVar setVar = new SetVar(SetType.SESSION, name, new BoolLiteral(value)); + Assertions.assertDoesNotThrow(() -> VariableMgr.setVar(variables, setVar)); + Assertions.assertDoesNotThrow(() -> VariableMgr.setVarForNonMasterFE(variables, setVar)); + } + VariableExpr expression = new VariableExpr(name); + VariableMgr.fillValue(variables, expression); + Assertions.assertEquals(Type.VARCHAR, expression.getType()); + Assertions.assertEquals("", VariableMgr.getValue(variables, expression)); + Literal literal = VariableMgr.getLiteral(variables, name, SetType.SESSION); + Assertions.assertNotNull(literal); + Assertions.assertEquals("", literal.getStringValue()); + Assertions.assertNull(VariableMgr.getVarContext(name)); + } + Assertions.assertFalse(VariableMgr.toMap(variables).containsKey("skip_checking_acid_version_file")); + } +} diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 6ea733a707d051..941c153f53190b 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -152,7 +152,7 @@ enum THdfsFileFormat { LZO_TEXT = 1, RC_FILE = 2, SEQUENCE_FILE =3, - AVRO = 4, + AVRO = 4, // Deprecated: reading this format is no longer supported. PARQUET = 5 } @@ -499,6 +499,7 @@ struct TTrinoConnectorTable { 3: optional map properties } +// Deprecated: the associated external table read support has been removed. struct TLakeSoulTable { 1: optional string db_name 2: optional string table_name @@ -538,7 +539,7 @@ struct TTableDescriptor { 20: optional TJdbcTable jdbcTable 21: optional TMCTable mcTable 22: optional TTrinoConnectorTable trinoConnectorTable - 23: optional TLakeSoulTable lakesoulTable + 23: optional TLakeSoulTable lakesoulTable // Deprecated: reading this format is no longer supported. 24: optional TDictionaryTable dictionaryTable 25: optional TRemoteDorisTable remoteDorisTable } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 4a84b7ed955900..cd02c8127ec903 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -124,7 +124,7 @@ enum TFileFormatType { FORMAT_JSON = 9, FORMAT_PROTO = 10, FORMAT_JNI = 11, - FORMAT_AVRO = 12, + FORMAT_AVRO = 12, // Deprecated: reading this format is no longer supported. FORMAT_CSV_LZ4BLOCK = 13, FORMAT_CSV_SNAPPYBLOCK = 14, FORMAT_WAL = 15, @@ -448,6 +448,7 @@ struct THudiFileDesc { 12: optional i64 schema_id; // for schema change. (native reader) } +// Deprecated: the associated external table read support has been removed. struct TLakeSoulFileDesc { 1: optional list file_paths; 2: optional list primary_keys; @@ -456,11 +457,13 @@ struct TLakeSoulFileDesc { 5: optional string options; } +// Deprecated: the associated external table read support has been removed. struct TTransactionalHiveDeleteDeltaDesc { 1: optional string directory_location 2: optional list file_names } +// Deprecated: the associated external table read support has been removed. struct TTransactionalHiveDesc { 1: optional string partition 2: optional list delete_deltas @@ -489,10 +492,10 @@ struct TTableFormatFileDesc { 2: optional TIcebergFileDesc iceberg_params 3: optional THudiFileDesc hudi_params 4: optional TPaimonFileDesc paimon_params - 5: optional TTransactionalHiveDesc transactional_hive_params + 5: optional TTransactionalHiveDesc transactional_hive_params // Deprecated: reading this format is no longer supported. 6: optional TMaxComputeFileDesc max_compute_params 7: optional TTrinoConnectorFileDesc trino_connector_params - 8: optional TLakeSoulFileDesc lakesoul_params + 8: optional TLakeSoulFileDesc lakesoul_params // Deprecated: reading this format is no longer supported. 9: optional i64 table_level_row_count = -1 10: optional TRemoteDorisFileDesc remote_doris_params // JDBC connection parameters (used when table_format_type == "jdbc") diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index e18c66ec89125a..ff06f95d7af46c 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -556,7 +556,7 @@ enum TTableType { JDBC_TABLE = 10, TEST_EXTERNAL_TABLE = 11, MAX_COMPUTE_TABLE = 12, - LAKESOUL_TABLE = 13, + LAKESOUL_TABLE = 13, // Deprecated: reading this format is no longer supported. TRINO_CONNECTOR_TABLE = 14, DICTIONARY_TABLE = 15, REMOTE_DORIS_TABLE = 16 diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 0baa216c1dced7..cb0a3a94d6f895 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -288,15 +288,6 @@ kerberosHdfsPort=8820 enableRefactorParamsHdfsTest=true -// LakeSoul catalog test config -enableLakesoulTest = false -lakesoulPGUser="*******" -lakesoulPGPwd="*******" -lakesoulPGUrl="*******" -lakesoulMinioAK="*******" -lakesoulMinioSK="*******" -lakesoulMinioEndpoint="*******" - // cloud metaServiceToken = "greedisgood9999" instanceId = "default_instance_id" diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out index 005f109060481b..207e787c850edd 100644 --- a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out +++ b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out @@ -1207,108 +1207,6 @@ true 127 1024 42 45097156608 3.1415 -2.713 \N \N \N \N \N 98 F W College 1000 Good 0 0 0 99 M U College 1000 Good 0 0 0 --- !test_32 -- -0 12 536870912 0 12 {"i":276, "j":605, "k":48} -0 12 536870912 1 12 {"i":700, "j":459, "k":443} -0 12 536870912 10 12 {"i":274, "j":96, "k":764} -0 12 536870912 11 12 {"i":808, "j":984, "k":271} -0 12 536870912 12 12 {"i":976, "j":552, "k":992} -0 12 536870912 13 12 {"i":17, "j":609, "k":884} -0 12 536870912 14 12 {"i":854, "j":406, "k":678} -0 12 536870912 15 12 {"i":332, "j":174, "k":239} -0 12 536870912 16 12 {"i":751, "j":656, "k":463} -0 12 536870912 17 12 {"i":170, "j":398, "k":265} -0 12 536870912 18 12 {"i":124, "j":776, "k":386} -0 12 536870912 19 12 {"i":715, "j":389, "k":836} -0 12 536870912 2 12 {"i":762, "j":18, "k":968} -0 12 536870912 20 12 {"i":235, "j":299, "k":140} -0 12 536870912 21 12 {"i":722, "j":872, "k":275} -0 12 536870912 22 12 {"i":332, "j":737, "k":315} -0 12 536870912 23 12 {"i":935, "j":844, "k":462} -0 12 536870912 24 12 {"i":214, "j":171, "k":728} -0 12 536870912 25 12 {"i":719, "j":134, "k":403} -0 12 536870912 26 12 {"i":920, "j":745, "k":973} -0 12 536870912 27 12 {"i":310, "j":338, "k":719} -0 12 536870912 28 12 {"i":71, "j":376, "k":619} -0 12 536870912 29 12 {"i":72, "j":899, "k":851} -0 12 536870912 3 12 {"i":810, "j":360, "k":994} -0 12 536870912 30 12 {"i":195, "j":106, "k":129} -0 12 536870912 31 12 {"i":105, "j":319, "k":601} -0 12 536870912 32 12 {"i":234, "j":729, "k":319} -0 12 536870912 33 12 {"i":262, "j":578, "k":371} -0 12 536870912 34 12 {"i":295, "j":158, "k":242} -0 12 536870912 35 12 {"i":850, "j":851, "k":584} -0 12 536870912 36 12 {"i":162, "j":61, "k":372} -0 12 536870912 37 12 {"i":50, "j":420, "k":929} -0 12 536870912 38 12 {"i":583, "j":305, "k":165} -0 12 536870912 39 12 {"i":439, "j":21, "k":659} -0 12 536870912 4 12 {"i":631, "j":299, "k":762} -0 12 536870912 40 12 {"i":892, "j":253, "k":781} -0 12 536870912 41 12 {"i":276, "j":220, "k":772} -0 12 536870912 42 12 {"i":940, "j":864, "k":359} -0 12 536870912 43 12 {"i":382, "j":431, "k":579} -0 12 536870912 44 12 {"i":560, "j":209, "k":104} -0 12 536870912 45 12 {"i":235, "j":386, "k":629} -0 12 536870912 46 12 {"i":924, "j":774, "k":135} -0 12 536870912 47 12 {"i":718, "j":285, "k":30} -0 12 536870912 48 12 {"i":960, "j":213, "k":835} -0 12 536870912 49 12 {"i":813, "j":660, "k":17} -0 12 536870912 5 12 {"i":24, "j":558, "k":124} -0 12 536870912 50 12 {"i":581, "j":303, "k":636} -0 12 536870912 51 12 {"i":774, "j":285, "k":179} -0 12 536870912 52 12 {"i":946, "j":912, "k":362} -0 12 536870912 53 12 {"i":709, "j":218, "k":70} -0 12 536870912 54 12 {"i":719, "j":507, "k":497} -0 12 536870912 55 12 {"i":5, "j":112, "k":882} -0 12 536870912 56 12 {"i":754, "j":608, "k":301} -0 12 536870912 57 12 {"i":980, "j":368, "k":697} -0 12 536870912 58 12 {"i":175, "j":664, "k":549} -0 12 536870912 59 12 {"i":599, "j":0, "k":663} -0 12 536870912 6 12 {"i":594, "j":30, "k":95} -0 12 536870912 60 12 {"i":920, "j":75, "k":943} -0 12 536870912 61 12 {"i":894, "j":5, "k":737} -0 12 536870912 62 12 {"i":745, "j":549, "k":359} -0 12 536870912 63 12 {"i":32, "j":523, "k":811} -0 12 536870912 64 12 {"i":465, "j":973, "k":171} -0 12 536870912 65 12 {"i":275, "j":762, "k":131} -0 12 536870912 66 12 {"i":758, "j":671, "k":206} -0 12 536870912 67 12 {"i":879, "j":572, "k":159} -0 12 536870912 68 12 {"i":9, "j":419, "k":400} -0 12 536870912 69 12 {"i":302, "j":534, "k":785} -0 12 536870912 7 12 {"i":782, "j":580, "k":536} -0 12 536870912 70 12 {"i":931, "j":378, "k":14} -0 12 536870912 71 12 {"i":201, "j":176, "k":326} -0 12 536870912 72 12 {"i":100, "j":697, "k":819} -0 12 536870912 73 12 {"i":245, "j":606, "k":265} -0 12 536870912 74 12 {"i":163, "j":13, "k":518} -0 12 536870912 75 12 {"i":952, "j":382, "k":262} -0 12 536870912 76 12 {"i":782, "j":6, "k":492} -0 12 536870912 77 12 {"i":494, "j":1, "k":22} -0 12 536870912 78 12 {"i":299, "j":930, "k":158} -0 12 536870912 79 12 {"i":7, "j":497, "k":514} -0 12 536870912 8 12 {"i":368, "j":225, "k":942} -0 12 536870912 80 12 {"i":560, "j":49, "k":609} -0 12 536870912 81 12 {"i":378, "j":761, "k":479} -0 12 536870912 82 12 {"i":268, "j":289, "k":733} -0 12 536870912 83 12 {"i":380, "j":601, "k":936} -0 12 536870912 84 12 {"i":722, "j":78, "k":87} -0 12 536870912 85 12 {"i":984, "j":742, "k":986} -0 12 536870912 86 12 {"i":414, "j":472, "k":265} -0 12 536870912 87 12 {"i":538, "j":342, "k":914} -0 12 536870912 88 12 {"i":881, "j":522, "k":464} -0 12 536870912 89 12 {"i":225, "j":732, "k":219} -0 12 536870912 9 12 {"i":916, "j":600, "k":70} -0 12 536870912 90 12 {"i":240, "j":594, "k":415} -0 12 536870912 91 12 {"i":33, "j":644, "k":707} -0 12 536870912 92 12 {"i":701, "j":128, "k":446} -0 12 536870912 93 12 {"i":532, "j":796, "k":318} -0 12 536870912 94 12 {"i":901, "j":401, "k":506} -0 12 536870912 95 12 {"i":164, "j":522, "k":425} -0 12 536870912 96 12 {"i":661, "j":302, "k":878} -0 12 536870912 97 12 {"i":854, "j":423, "k":597} -0 12 536870912 98 12 {"i":449, "j":961, "k":176} -0 12 536870912 99 12 {"i":614, "j":717, "k":177} - -- !test_34 -- 1 1a 10 10a diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 2af49b8fd63c6a..43f54b6ca921e1 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -212,9 +212,6 @@ kerberosHdfsPort=8820 enableNonCatalogKerberosTest = true -// LakeSoul catalog test config -enableLakesoulTest = true - // AWS iam role config // hudi p0 external regression test config diff --git a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy index 8c2a55d9e9dffa..2723cb431cc701 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy @@ -116,9 +116,6 @@ suite("test_hive_basic_type", "p0,external") { // At present, doris only supports three formats of orc parquet textfile, while others are not supported - // hive tables in avro format are not supported - //order_qt_34 """select * from ${catalog_name}.${ex_db_name}.avro_all_types limit 1;""" - // hive tables in SEQUENCEFILE format are not supported //order_qt_35 """select * from ${catalog_name}.${ex_db_name}.sequence_all_types limit 1;""" diff --git a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy index 7775781ea3bd7a..3c4a24cfe56d4f 100644 --- a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy +++ b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy @@ -215,12 +215,6 @@ suite("test_hdfs_orc_group0_orc_files", "p0,external") { "format" = "orc") order by _col0 limit 100; """ - uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/acid5k.orc" - order_qt_test_32 """ select * from HDFS( - "uri" = "${uri}", - "hadoop.username" = "${hdfsUserName}", - "format" = "orc") order by rowid limit 100; """ - uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/orc_index_int_string.orc" order_qt_test_34 """ select * from HDFS( "uri" = "${uri}", diff --git a/samples/datalake/lakesoul/README.md b/samples/datalake/lakesoul/README.md deleted file mode 100644 index 136c249fbe0c4f..00000000000000 --- a/samples/datalake/lakesoul/README.md +++ /dev/null @@ -1,277 +0,0 @@ -# doris-lakesoul-compose -## Launch Docker Compose - -First, we need to ensure the environmental parameters of the machine. - -``` -sysctl -w vm.max_map_count=2000000 -``` - -We can then start all the required containers via the script. - -``` -bash start_all.sh -``` -It will start a set of docker, the environment includes: -- doris -- postgreSQL -- mysql -- flink -- spark -- minIO - -And it will automatically create an lakesoul table. We can use these tables directly to experience doris. - -## LakeSoul Table QuickStart - -Now we can query this table through doris. - -``` -bash start_doris_client.sh -``` - -After entering the doris client, the lakesoul catalog has been created here, so the data of the lakesoul table can be directly queried. - -```sql -mysql> show catalogs; -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -| CatalogId | CatalogName | Type | IsCurrent | CreateTime | LastUpdateTime | Comment | -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -| 0 | internal | internal | Yes | NULL | NULL | Doris internal catalog | -| 10003 | lakesoul | lakesoul | No | 2024-11-08 05:47:59.304898431 | NULL | | -+-----------+-------------+----------+-----------+-------------------------------+----------------+------------------------+ -2 rows in set (0.04 sec) - -mysql> use lakesoul.tpch; -Reading table information for completion of table and column names -You can turn off this feature to get a quicker startup with -A - -Database changed - -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer_from_spark | -+---------------------+ -1 row in set (0.00 sec) - -mysql> select * from customer_from_spark where c_nationkey = 1 order by c_custkey limit 4; -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 14 | Customer#000000014 | KXkletMlL2JQEA | 1 | 11-845-129-3851 | 5266.30 | FURNITURE | , ironic packages across the unus | -| 30 | Customer#000000030 | nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY | 1 | 11-764-165-5076 | 9321.01 | BUILDING | lithely final requests. furiously unusual account | -| 59 | Customer#000000059 | zLOCP0wh92OtBihgspOGl4 | 1 | 11-355-584-3112 | 3458.60 | MACHINERY | ously final packages haggle blithely after the express deposits. furiou | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+--------------------------------------------------------------------------------------------------------+ -4 rows in set (3.14 sec) - -mysql> select * from customer_from_spark where c_nationkey = 1 order by c_custkey desc limit 4; -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -| 14983 | Customer#000014983 | ERN3vq5Fvt4DL | 1 | 11-424-279-1846 | 841.22 | AUTOMOBILE | furiously slyly special foxes. express theodolites cajole carefully. special dinos haggle pinto | -| 14968 | Customer#000014968 | ,sykKTZBzVFl7ito1750v2TRYwmkRl2nvqGHwmx | 1 | 11-669-222-9657 | 6106.77 | HOUSEHOLD | ts above the furiously even deposits haggle across | -| 14961 | Customer#000014961 | JEIORcsBp6RpLYH 9gNdDyWJ | 1 | 11-490-251-5554 | 4006.35 | HOUSEHOLD | quests detect carefully final platelets! quickly final frays haggle slyly blithely final acc | -| 14940 | Customer#000014940 | bNoyCxPuqSwPLjbqjEUNGN d0mSP | 1 | 11-242-677-1085 | 8829.48 | HOUSEHOLD | ver the quickly express braids. regular dependencies haggle fluffily quickly i | -+-----------+--------------------+-----------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------+ -4 rows in set (0.10 sec) -``` - -Doris can perform partition pruning on LakeSoul and speed up the query process through native reading. We can check this through `explain verbose`. - -```sql -mysql> explain verbose select * from customer_from_spark where c_nationkey < 3; -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Explain String(Old Planner) | -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| PLAN FRAGMENT 0 | -| OUTPUT EXPRS: | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_custkey` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_name` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_address` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_nationkey` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_phone` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_acctbal` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_mktsegment` | -| `lakesoul`.`tpch`.`customer_from_spark`.`c_comment` | -| PARTITION: UNPARTITIONED | -| | -| HAS_COLO_PLAN_NODE: false | -| | -| VRESULT SINK | -| MYSQL_PROTOCOL | -| | -| 1:VEXCHANGE | -| offset: 0 | -| tuple ids: 0 | -| | -| PLAN FRAGMENT 1 | -| | -| PARTITION: RANDOM | -| | -| HAS_COLO_PLAN_NODE: false | -| | -| STREAM DATA SINK | -| EXCHANGE ID: 01 | -| UNPARTITIONED | -| | -| 0:VplanNodeName | -| table: customer_from_spark | -| predicates: (`c_nationkey` < 3) | -| inputSplitNum=12, totalFileSize=0, scanRanges=12 | -| partition=0/0 | -| backends: | -| 10002 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00000-0568c817-d6bc-4fa1-bb9e-b311069b131c_00000.c000.parquet start: 0 length: 0 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00001-d99a8fe6-61ab-4285-94da-2f84f8746a8a_00001.c000.parquet start: 0 length: 0 | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=1/part-00002-8a8e396f-685f-4b0f-87fa-e2a3fe5be87e_00002.c000.parquet start: 0 length: 0 | -| ... other 8 files ... | -| s3://lakesoul-test-bucket/data/tpch/customer_from_spark/c_nationkey=0/part-00003-d5b598cd-5bed-412c-a26f-bb4bc9c937bc_00003.c000.parquet start: 0 length: 0 | -| numNodes=1 | -| pushdown agg=NONE | -| tuple ids: 0 | -| | -| Tuples: | -| TupleDescriptor{id=0, tbl=customer_from_spark} | -| SlotDescriptor{id=0, col=c_custkey, colUniqueId=0, type=int, nullable=false, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=1, col=c_name, colUniqueId=1, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=2, col=c_address, colUniqueId=2, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=3, col=c_nationkey, colUniqueId=3, type=int, nullable=false, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=4, col=c_phone, colUniqueId=4, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=5, col=c_acctbal, colUniqueId=5, type=decimalv3(15,2), nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=6, col=c_mktsegment, colUniqueId=6, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -| SlotDescriptor{id=7, col=c_comment, colUniqueId=7, type=text, nullable=true, isAutoIncrement=false, subColPath=null} | -+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -57 rows in set (0.03 sec) - -``` - -## LakeSoul CDC Table - -Launch Flink CDC Job to sync mysql table. The mysql table is loaded when launching docker compose. - -``` -bash start_flink_cdc_job.sh -``` - -```sql -Start flink-cdc job... -SLF4J: Class path contains multiple SLF4J bindings. -SLF4J: Found binding in [jar:file:/opt/flink/lib/log4j-slf4j-impl-2.17.1.jar!/org/slf4j/impl/StaticLoggerBinder.class] -SLF4J: Found binding in [jar:file:/opt/hadoop/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class] -SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. -SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] -Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary. -Job has been submitted with JobID d1b3641dcd1ad85c6b373d49b1867e68 - -``` - - -Flink CDC Job will be launched. We can check the process of launching at `doris client` by recreate the lakesoul catalog. After the Flink CDC Job has been launched, we can see the syncing LakeSoul CDC table at `doris client`. - -```sql -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer_from_spark | -+---------------------+ -2 rows in set (0.00 sec) - - -mysql> drop catalog if exists lakesoul; -Query OK, 0 rows affected (0.00 sec) - -mysql> create catalog `lakesoul` properties ('type'='lakesoul', 'lakesoul.pg.username'='lakesoul_test', 'lakesoul.pg.password'='lakesoul_test', 'lakesoul.pg.url'='jdbc:postgresql://lakesoul-meta-pg:5432/lakesoul_test?stringtype=unspecified', 'minio.endpoint'='http://minio:9000', 'minio.access_key'='admin', 'minio.secret_key'='password'); -Query OK, 0 rows affected (0.01 sec) - -mysql> show tables; -+---------------------+ -| Tables_in_tpch | -+---------------------+ -| customer | -| customer_from_spark | -+---------------------+ -2 rows in set (0.00 sec) - -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 711.56 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -9 rows in set (1.09 sec) - -``` - -Enter the `mysql client` and try to modify data. - -``` -bash start_mysql_client.sh -``` - -Try update row from `mysql client`. - -```sql -mysql> update customer set c_acctbal=2211.26 where c_custkey=1; -Query OK, 1 row affected (0.01 sec) -Rows matched: 1 Changed: 1 Warnings: 0 -``` - -Back to `doris client` and check the data changing. - -```sql -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 2 | Customer#000000002 | XSTf4,NCwDVaWNe6tEgvwfmRchLXak | 13 | 23-768-687-3665 | 121.65 | AUTOMOBILE | l accounts. blithely ironic theodolites integrate boldly: caref | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 2211.26 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -9 rows in set (0.11 sec) - -``` - -Try delete row from `mysql client`. - -```sql -mysql> delete from customer where c_custkey = 2; -Query OK, 1 row affected (0.01 sec) -``` - -Back to `doris client` and check the data changing. - -```sql -mysql> select c_custkey, c_name, c_address, c_nationkey , c_phone, c_acctbal , c_mktsegment , c_comment from lakesoul.tpch.customer where c_custkey < 10; -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -| 6 | Customer#000000006 | sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn | 20 | 30-114-968-4951 | 7638.57 | AUTOMOBILE | tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious | -| 9 | Customer#000000009 | xKiAFTjUsCuxfeleNqefumTrjS | 8 | 18-338-906-3675 | 8324.07 | FURNITURE | r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl | -| 1 | Customer#000000001 | IVhzIApeRb ot,c,E | 15 | 25-989-741-2988 | 2211.26 | BUILDING | to the even, regular platelets. regular, ironic epitaphs nag e | -| 3 | Customer#000000003 | MG9kdTD2WBHm | 1 | 11-719-748-3364 | 7498.12 | AUTOMOBILE | deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov | -| 7 | Customer#000000007 | TcGe5gaZNgVePxU5kRrvXBfkasDTea | 18 | 28-190-982-9759 | 9561.95 | AUTOMOBILE | ainst the ironic, express theodolites. express, even pinto beans among the exp | -| 8 | Customer#000000008 | I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 | 17 | 27-147-574-9335 | 6819.74 | BUILDING | among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide | -| 4 | Customer#000000004 | XxVSJsLAGtn | 4 | 14-128-190-5944 | 2866.83 | MACHINERY | requests. final, regular ideas sleep final accou | -| 5 | Customer#000000005 | KvpyuHCplrB84WgAiGV6sYpZq7Tj | 3 | 13-750-942-6364 | 794.47 | HOUSEHOLD | n accounts will have to unwind. foxes cajole accor | -+-----------+--------------------+---------------------------------------+-------------+-----------------+-----------+--------------+-------------------------------------------------------------------------------------------------------------------+ -8 rows in set (0.11 sec) - -``` diff --git a/samples/datalake/lakesoul/conf/my.conf b/samples/datalake/lakesoul/conf/my.conf deleted file mode 100644 index 8a4f2b1acd6edd..00000000000000 --- a/samples/datalake/lakesoul/conf/my.conf +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -[mysqld] -pid-file = /var/run/mysqld/mysqld.pid -socket = /var/run/mysqld/mysqld.sock -datadir = /var/lib/mysql -secure-file-priv= /data/tpch/ -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 - -# Custom config should go here -!includedir /etc/mysql/conf.d/ \ No newline at end of file diff --git a/samples/datalake/lakesoul/conf/properties b/samples/datalake/lakesoul/conf/properties deleted file mode 100644 index dceb3d5d11e5e8..00000000000000 --- a/samples/datalake/lakesoul/conf/properties +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# table_num 生成表数据量 -table_num=10 -# host mysql域名 -host=localhost -# user: mysql账号; password: mysql密码; port: mysql端口号; db: mysql数据库名; -user=root -password=root -port=3306 -db=test_cdc -timezone=UTC -# row_num随机向表中插入多少条数据 -row_num=500 -# delete_num随机删除多少条数据 -delete_num=50 diff --git a/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template b/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template deleted file mode 100644 index 385b2e772d2c80..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/fairscheduler.xml.template +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - FAIR - 1 - 2 - - - FIFO - 2 - 3 - - diff --git a/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template b/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template deleted file mode 100644 index ab96e03baed203..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/log4j2.properties.template +++ /dev/null @@ -1,69 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the console -rootLogger.level = info -rootLogger.appenderRef.stdout.ref = console - -# In the pattern layout configuration below, we specify an explicit `%ex` conversion -# pattern for logging Throwables. If this was omitted, then (by default) Log4J would -# implicitly add an `%xEx` conversion pattern which logs stacktraces with additional -# class packaging information. That extra information can sometimes add a substantial -# performance overhead, so we disable it in our default logging config. -# For more information, see SPARK-39361. -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex - -# Set the default spark-shell/spark-sql log level to WARN. When running the -# spark-shell/spark-sql, the log level for these classes is used to overwrite -# the root logger's log level, so that the user can have different defaults -# for the shell and regular Spark apps. -logger.repl.name = org.apache.spark.repl.Main -logger.repl.level = warn - -logger.thriftserver.name = org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver -logger.thriftserver.level = warn - -# Settings to quiet third party logs that are too verbose -logger.jetty1.name = org.sparkproject.jetty -logger.jetty1.level = warn -logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle -logger.jetty2.level = error -logger.replexprTyper.name = org.apache.spark.repl.SparkIMain$exprTyper -logger.replexprTyper.level = info -logger.replSparkILoopInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter -logger.replSparkILoopInterpreter.level = info -logger.parquet1.name = org.apache.parquet -logger.parquet1.level = error -logger.parquet2.name = parquet -logger.parquet2.level = error - -# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support -logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler -logger.RetryingHMSHandler.level = fatal -logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry -logger.FunctionRegistry.level = error - -# For deploying Spark ThriftServer -# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 -appender.console.filter.1.type = RegexFilter -appender.console.filter.1.regex = .*Thrift error occurred during processing of message.* -appender.console.filter.1.onMatch = deny -appender.console.filter.1.onMismatch = neutral diff --git a/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template b/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template deleted file mode 100644 index f52d33fd642230..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/metrics.properties.template +++ /dev/null @@ -1,210 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# syntax: [instance].sink|source.[name].[options]=[value] - -# This file configures Spark's internal metrics system. The metrics system is -# divided into instances which correspond to internal components. -# Each instance can be configured to report its metrics to one or more sinks. -# Accepted values for [instance] are "master", "worker", "executor", "driver", -# and "applications". A wildcard "*" can be used as an instance name, in -# which case all instances will inherit the supplied property. -# -# Within an instance, a "source" specifies a particular set of grouped metrics. -# there are two kinds of sources: -# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will -# collect a Spark component's internal state. Each instance is paired with a -# Spark source that is added automatically. -# 2. Common sources, like JvmSource, which will collect low level state. -# These can be added through configuration options and are then loaded -# using reflection. -# -# A "sink" specifies where metrics are delivered to. Each instance can be -# assigned one or more sinks. -# -# The sink|source field specifies whether the property relates to a sink or -# source. -# -# The [name] field specifies the name of source or sink. -# -# The [options] field is the specific property of this source or sink. The -# source or sink is responsible for parsing this property. -# -# Notes: -# 1. To add a new sink, set the "class" option to a fully qualified class -# name (see examples below). -# 2. Some sinks involve a polling period. The minimum allowed polling period -# is 1 second. -# 3. Wildcard properties can be overridden by more specific properties. -# For example, master.sink.console.period takes precedence over -# *.sink.console.period. -# 4. A metrics specific configuration -# "spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties" should be -# added to Java properties using -Dspark.metrics.conf=xxx if you want to -# customize metrics system. You can also put the file in ${SPARK_HOME}/conf -# and it will be loaded automatically. -# 5. The MetricsServlet sink is added by default as a sink in the master, -# worker and driver, and you can send HTTP requests to the "/metrics/json" -# endpoint to get a snapshot of all the registered metrics in JSON format. -# For master, requests to the "/metrics/master/json" and -# "/metrics/applications/json" endpoints can be sent separately to get -# metrics snapshots of the master instance and applications. This -# MetricsServlet does not have to be configured. -# 6. The metrics system can also be configured using Spark configuration -# parameters. The relevant parameter names are formed by adding the -# prefix "spark.metrics.conf." to the configuration entries detailed in -# this file (see examples below). - -## List of available common sources and their properties. - -# org.apache.spark.metrics.source.JvmSource -# Note: Currently, JvmSource is the only available common source. -# It can be added to an instance by setting the "class" option to its -# fully qualified class name (see examples below). - -## List of available sinks and their properties. - -# org.apache.spark.metrics.sink.ConsoleSink -# Name: Default: Description: -# period 10 Poll period -# unit seconds Unit of the poll period - -# org.apache.spark.metrics.sink.CSVSink -# Name: Default: Description: -# period 10 Poll period -# unit seconds Unit of the poll period -# directory /tmp Where to store CSV files - -# org.apache.spark.metrics.sink.GangliaSink -# Name: Default: Description: -# host NONE Hostname or multicast group of the Ganglia server, -# must be set -# port NONE Port of the Ganglia server(s), must be set -# period 10 Poll period -# unit seconds Unit of the poll period -# ttl 1 TTL of messages sent by Ganglia -# dmax 0 Lifetime in seconds of metrics (0 never expired) -# mode multicast Ganglia network mode ('unicast' or 'multicast') - -# org.apache.spark.metrics.sink.JmxSink - -# org.apache.spark.metrics.sink.MetricsServlet -# Name: Default: Description: -# path VARIES* Path prefix from the web server root -# sample false Whether to show entire set of samples for histograms -# ('false' or 'true') -# -# * Default path is /metrics/json for all instances except the master. The -# master has two paths: -# /metrics/applications/json # App information -# /metrics/master/json # Master information - -# org.apache.spark.metrics.sink.PrometheusServlet -# Name: Default: Description: -# path VARIES* Path prefix from the web server root -# -# * Default path is /metrics/prometheus for all instances except the master. The -# master has two paths: -# /metrics/applications/prometheus # App information -# /metrics/master/prometheus # Master information - -# org.apache.spark.metrics.sink.GraphiteSink -# Name: Default: Description: -# host NONE Hostname of the Graphite server, must be set -# port NONE Port of the Graphite server, must be set -# period 10 Poll period -# unit seconds Unit of the poll period -# prefix EMPTY STRING Prefix to prepend to every metric's name -# protocol tcp Protocol ("tcp" or "udp") to use -# regex NONE Optional filter to send only metrics matching this regex string - -# org.apache.spark.metrics.sink.StatsdSink -# Name: Default: Description: -# host 127.0.0.1 Hostname or IP of StatsD server -# port 8125 Port of StatsD server -# period 10 Poll period -# unit seconds Units of poll period -# prefix EMPTY STRING Prefix to prepend to metric name - -## Examples -# Enable JmxSink for all instances by class name -#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink - -# Enable ConsoleSink for all instances by class name -#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink - -# Enable StatsdSink for all instances by class name -#*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink -#*.sink.statsd.prefix=spark - -# Polling period for the ConsoleSink -#*.sink.console.period=10 -# Unit of the polling period for the ConsoleSink -#*.sink.console.unit=seconds - -# Polling period for the ConsoleSink specific for the master instance -#master.sink.console.period=15 -# Unit of the polling period for the ConsoleSink specific for the master -# instance -#master.sink.console.unit=seconds - -# Enable CsvSink for all instances by class name -#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink - -# Polling period for the CsvSink -#*.sink.csv.period=1 -# Unit of the polling period for the CsvSink -#*.sink.csv.unit=minutes - -# Polling directory for CsvSink -#*.sink.csv.directory=/tmp/ - -# Polling period for the CsvSink specific for the worker instance -#worker.sink.csv.period=10 -# Unit of the polling period for the CsvSink specific for the worker instance -#worker.sink.csv.unit=minutes - -# Enable Slf4jSink for all instances by class name -#*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink - -# Polling period for the Slf4JSink -#*.sink.slf4j.period=1 -# Unit of the polling period for the Slf4jSink -#*.sink.slf4j.unit=minutes - -# Example configuration for Graphite sink -#*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink -#*.sink.graphite.host= -#*.sink.graphite.port= -#*.sink.graphite.period=10 -#*.sink.graphite.unit=seconds -#*.sink.graphite.prefix= - -# Enable JvmSource for instance master, worker, driver and executor -#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource - -# Example configuration for PrometheusServlet -#*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet -#*.sink.prometheusServlet.path=/metrics/prometheus -#master.sink.prometheusServlet.path=/metrics/master/prometheus -#applications.sink.prometheusServlet.path=/metrics/applications/prometheus diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf b/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf deleted file mode 100755 index 844709658bcde0..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Default system properties included when running spark-submit. -# This is useful for setting default environmental settings. - -# Example: -spark.sql.extensions com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension -spark.sql.catalog.lakesoul org.apache.spark.sql.lakesoul.catalog.LakeSoulCatalog -# spark.sql.defaultCatalog lakesoul -# spark.eventLog.enabled true -# spark.sql.catalogImplementation in-memory diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template b/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template deleted file mode 100644 index 19cba6e71ed190..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-defaults.conf.template +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Default system properties included when running spark-submit. -# This is useful for setting default environmental settings. - -# Example: -# spark.master spark://master:7077 -# spark.eventLog.enabled true -# spark.eventLog.dir hdfs://namenode:8021/directory -# spark.serializer org.apache.spark.serializer.KryoSerializer -# spark.driver.memory 5g -# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" diff --git a/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template b/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template deleted file mode 100755 index e9491995e727eb..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/spark-env.sh.template +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file is sourced when running various Spark programs. -# Copy it as spark-env.sh and edit that to configure Spark for your site. - -# Options read when launching programs locally with -# ./bin/run-example or ./bin/spark-submit -# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files -# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node -# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program - -# Options read by executors and drivers running inside the cluster -# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node -# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program -# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data -# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos - -# Options read in any mode -# - SPARK_CONF_DIR, Alternate conf dir. (Default: ${SPARK_HOME}/conf) -# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). -# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) -# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) - -# Options read in any cluster manager using HDFS -# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files - -# Options read in YARN client/cluster mode -# - YARN_CONF_DIR, to point Spark towards YARN configuration files when you use YARN - -# Options for the daemons used in the standalone deploy mode -# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname -# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master -# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") -# - SPARK_WORKER_CORES, to set the number of cores to use on this machine -# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) -# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker -# - SPARK_WORKER_DIR, to set the working directory of worker processes -# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") -# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). -# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") -# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") -# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") -# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons -# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers - -# Options for launcher -# - SPARK_LAUNCHER_OPTS, to set config properties and Java options for the launcher (e.g. "-Dx=y") - -# Generic options for the daemons used in the standalone deploy mode -# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) -# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) -# - SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5. -# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) -# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) -# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) -# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. -# Options for native BLAS, like Intel MKL, OpenBLAS, and so on. -# You might get better performance to enable these options if using native BLAS (see SPARK-21305). -# - MKL_NUM_THREADS=1 Disable multi-threading of Intel MKL -# - OPENBLAS_NUM_THREADS=1 Disable multi-threading of OpenBLAS - -# Options for beeline -# - SPARK_BEELINE_OPTS, to set config properties only for the beeline cli (e.g. "-Dx=y") -# - SPARK_BEELINE_MEMORY, Memory for beeline (e.g. 1000M, 2G) (Default: 1G) diff --git a/samples/datalake/lakesoul/data/spark-conf/workers.template b/samples/datalake/lakesoul/data/spark-conf/workers.template deleted file mode 100644 index be42a638230b78..00000000000000 --- a/samples/datalake/lakesoul/data/spark-conf/workers.template +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# A Spark Worker will be started on each of the machines listed below. -localhost \ No newline at end of file diff --git a/samples/datalake/lakesoul/docker-compose.env b/samples/datalake/lakesoul/docker-compose.env deleted file mode 100644 index 9485df04ae20a5..00000000000000 --- a/samples/datalake/lakesoul/docker-compose.env +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -PG_PORT=29013 -MYSQL_PORT=29014 -MINIO_UI_PORT=29002 -MINIO_API_PORT=29001 -DORIS_QUERY_PORT=29031 -FLINK_JOBMANAGER_PORT=28081 -LAKESOUL_META_HOST=lakesoul-meta-pg \ No newline at end of file diff --git a/samples/datalake/lakesoul/docker-compose.yml b/samples/datalake/lakesoul/docker-compose.yml deleted file mode 100644 index affe98e3fe6c17..00000000000000 --- a/samples/datalake/lakesoul/docker-compose.yml +++ /dev/null @@ -1,214 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3' - -services: - postgres: - image: postgres:14.5 - container_name: doris-lakesoul-pg - hostname: ${LAKESOUL_META_HOST} - networks: - - doris-lakesoul - ports: - - "${PG_PORT}:5432" - restart: always - environment: - POSTGRES_PASSWORD: lakesoul_test - POSTGRES_USER: lakesoul_test - POSTGRES_DB: lakesoul_test - command: - - "postgres" - - "-c" - - "max_connections=4096" - - "-c" - - "default_transaction_isolation=serializable" - volumes: - - ./sql/meta_init.sql:/docker-entrypoint-initdb.d/meta_init.sql - - ./sql/meta_cleanup.sql:/meta_cleanup.sql - - doris: - image: mysql:8.0.18 - container_name: doris-lakesoul-doris - networks: - - doris-lakesoul - hostname: doris - user: root - privileged: true - ports: - - ${DORIS_QUERY_PORT}:9030 - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - LD_LIBRARY_PATH=/opt/doris/be/lib - - JAVA_HOME=/opt/jdk-17 - volumes: - - ./packages/jdk-17:/opt/jdk-17 - - ./packages/doris-bin:/opt/doris-bin - - ./sql/init_doris.sql:/opt/doris-bin/init_doris.sql - - ./scripts:/opt/scripts - command: bin/bash /opt/scripts/start_doris.sh - - mysql: - image: mysql:8.0.18 - restart: always - container_name: doris-lakesoul-mysql-source - hostname: mysql - user: root - networks: - - doris-lakesoul - ports: - - ${MYSQL_PORT}:3306 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: tpch - command: - --max_connections=4096 - volumes: - - ./sql/mysql_init.sql:/docker-entrypoint-initdb.d/mysql_init.sql - - ./scripts:/opt/scripts - - ./conf/properties:/properties - - ./conf/my.conf:/etc/mysql/my.cnf - - ./packages/tpch-dbgen/customer.tbl:/data/tpch/customer.tbl - - minio: - image: bitnami/minio:latest - container_name: doris-lakesoul-minio - ports: - - ${MINIO_API_PORT}:9000 - - ${MINIO_UI_PORT}:9001 - environment: - MINIO_DEFAULT_BUCKETS: lakesoul-test-bucket:public - MINIO_ROOT_USER: admin - MINIO_ROOT_PASSWORD: password - MINIO_DOMAIN: minio - healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] - interval: 3s - timeout: 5s - retries: 3 - hostname: minio - networks: - - doris-lakesoul - # profiles: ["s3"] - - jobmanager: - image: flink:1.17.1-scala_2.12-java8 - container_name: doris-lakesoul-jobmanager - ports: - - ${FLINK_JOBMANAGER_PORT}:8081 - command: jobmanager - networks: - - doris-lakesoul - depends_on: - - postgres - - minio - deploy: - replicas: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.memory.process.size: 3000m - taskmanager.memory.flink.size: 2048m - s3.endpoint: http://minio:9000 - s3.access-key: admin - s3.secret-key: password - s3.path.style.access: true - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - - FLINK_HOME=/opt/flink - - HADOOP_HOME=/opt/hadoop - - HADOOP_CLASSPATH="/opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/yarn:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*" - volumes: - - ./work-dir/:/opt/flink/work-dir/ - - ./packages/hadoop-3.3.5:/opt/hadoop - - ./packages/jars/lakesoul-flink-1.17-2.6.1.jar:/opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar - - ./packages/jars/flink-s3-fs-hadoop-1.17.1.jar:/opt/flink/plugins/s3/flink-s3-fs-hadoop-1.17.1.jar - - ./packages/jars/flink-parquet-1.17.1.jar:/opt/flink/lib/flink-parquet-1.17.1.jar - - ./packages/jars/parquet-hadoop-bundle-1.12.3.jar:/opt/flink/lib/parquet-hadoop-bundle-1.12.3.jar - - ./sql/init_flink_tables.sql:/opt/flink/sql/init_tables.sql - - - taskmanager: - image: flink:1.17.1-scala_2.12-java8 - depends_on: - - jobmanager - command: taskmanager - deploy: - replicas: 2 - networks: - - doris-lakesoul - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.memory.process.size: 3000m - taskmanager.memory.flink.size: 2048m - taskmanager.numberOfTaskSlots: 6 - s3.endpoint: http://minio:9000 - s3.access-key: admin - s3.secret-key: password - s3.path.style.access: true - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - - FLINK_HOME=/opt/flink - - HADOOP_HOME=/opt/hadoop - - HADOOP_CLASSPATH="/opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/yarn:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*" - volumes: - - ./work-dir/:/opt/flink/work-dir/ - - ./packages/hadoop-3.3.5:/opt/hadoop - - ./packages/jars/lakesoul-flink-1.17-2.6.1.jar:/opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar - - ./packages/jars/flink-s3-fs-hadoop-1.17.1.jar:/opt/flink/plugins/s3/flink-s3-fs-hadoop-1.17.1.jar - - ./packages/jars/flink-parquet-1.17.1.jar:/opt/flink/lib/flink-parquet-1.17.1.jar - - ./packages/jars/parquet-hadoop-bundle-1.12.3.jar:/opt/flink/lib/parquet-hadoop-bundle-1.12.3.jar - - spark: - image: bitnami/spark:3.3.1 - container_name: doris-lakesoul-spark - build: spark/ - privileged: true - user: root - volumes: - - ./packages/jars/lakesoul-spark-3.3-2.6.1.jar:/opt/bitnami/spark/jars/lakesoul-spark-3.3-2.6.1.jar - - ./data/spark-conf:/opt/bitnami/spark/conf - - ./sql/prepare_data.sql:/opt/sql/prepare_data.sql - - ./packages/tpch-dbgen/customer.tbl:/data/tpch/customer.tbl - - ./packages/tpch-dbgen/orders.tbl:/data/tpch/orders.tbl - depends_on: - - postgres - - minio - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - LAKESOUL_PG_DRIVER=com.lakesoul.shaded.org.postgresql.Driver - - LAKESOUL_PG_URL=jdbc:postgresql://${LAKESOUL_META_HOST}:5432/lakesoul_test?stringtype=unspecified - - LAKESOUL_PG_USERNAME=lakesoul_test - - LAKESOUL_PG_PASSWORD=lakesoul_test - networks: - - doris-lakesoul - -networks: - doris-lakesoul: - ipam: - driver: default \ No newline at end of file diff --git a/samples/datalake/lakesoul/scripts/load_tpch_data.sh b/samples/datalake/lakesoul/scripts/load_tpch_data.sh deleted file mode 100644 index bd1534d0912a2f..00000000000000 --- a/samples/datalake/lakesoul/scripts/load_tpch_data.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -git clone https://github.com/databricks/tpch-dbgen.git -cd tpch-dbgen -make -./dbgen -f -s 0.1 \ No newline at end of file diff --git a/samples/datalake/lakesoul/scripts/start_doris.sh b/samples/datalake/lakesoul/scripts/start_doris.sh deleted file mode 100644 index ef82cc0e7dfd88..00000000000000 --- a/samples/datalake/lakesoul/scripts/start_doris.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export JAVA_HOME=/opt/jdk-17 -export PATH=${JAVA_HOME}/bin:${PATH} - -echo 'start to copy...' -cp -r /opt/doris-bin /opt/doris - -echo 'start fe...' -rm -rf /opt/doris/fe/doris-meta/* -/opt/doris/fe/bin/start_fe.sh --daemon - -echo 'start be...' -rm -rf /opt/doris/be/storage/* -/opt/doris/be/bin/start_be.sh --daemon - -while [[ ! -f "/opt/doris/fe/log/fe.log" ]]; do - echo "wait log..." - sleep 2 -done - -QE=$(grep "QE service start." /opt/doris/fe/log/fe.log) -while [[ -z "${QE}" ]]; do - echo "wait fe..." - sleep 2 - QE=$(grep "QE service start." /opt/doris/fe/log/fe.log) -done - -echo 'doris is started' - -MYSQL_ERROR=$(mysql -u root -P 9030 -h doris &1) -ERR=$(echo "${MYSQL_ERROR}" | grep "Can't connect to MySQL") -echo "${ERR}" -while [[ -n "${ERR}" ]]; do - echo "wait mysql..." - sleep 2 - MYSQL_ERROR=$(mysql -u root -P 9030 -h doris &1) - ERR=$(echo "${MYSQL_ERROR}" | grep "Can't connect to MySQL") -done - -echo 'doris is inited' - -tail -F /dev/null diff --git a/samples/datalake/lakesoul/sql/init_doris.sql b/samples/datalake/lakesoul/sql/init_doris.sql deleted file mode 100644 index ead15ec80e5aca..00000000000000 --- a/samples/datalake/lakesoul/sql/init_doris.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER SYSTEM ADD BACKEND '127.0.0.1:9050'; - -drop catalog if exists lakesoul; - -create catalog `lakesoul` properties ( - 'type'='lakesoul', - 'lakesoul.pg.username'='lakesoul_test', - 'lakesoul.pg.password'='lakesoul_test', - 'lakesoul.pg.url'='jdbc:postgresql://lakesoul-meta-pg:5432/lakesoul_test?stringtype=unspecified', - 'minio.endpoint'='http://minio:9000', - 'minio.access_key'='admin', - 'minio.secret_key'='password' - ); \ No newline at end of file diff --git a/samples/datalake/lakesoul/sql/meta_cleanup.sql b/samples/datalake/lakesoul/sql/meta_cleanup.sql deleted file mode 100644 index dd25a7ee958879..00000000000000 --- a/samples/datalake/lakesoul/sql/meta_cleanup.sql +++ /dev/null @@ -1,11 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -delete from namespace; -insert into namespace(namespace, properties, comment) values ('default', '{}', ''); -delete from data_commit_info; -delete from table_info; -delete from table_path_id; -delete from table_name_id; -delete from partition_info; diff --git a/samples/datalake/lakesoul/sql/meta_init.sql b/samples/datalake/lakesoul/sql/meta_init.sql deleted file mode 100644 index 28b4eca4b6c164..00000000000000 --- a/samples/datalake/lakesoul/sql/meta_init.sql +++ /dev/null @@ -1,145 +0,0 @@ --- SPDX-FileCopyrightText: 2023 LakeSoul Contributors --- --- SPDX-License-Identifier: Apache-2.0 - -create table if not exists namespace -( - namespace text, - properties json, - comment text, - domain text default 'public', - primary key (namespace) -); - -insert into namespace(namespace, properties, comment) values ('default', '{}', '') -ON CONFLICT DO NOTHING; - -create table if not exists table_info -( - table_id text, - table_namespace text default 'default', - table_name text, - table_path text, - table_schema text, - properties json, - partitions text, - domain text default 'public', - primary key (table_id) -); - -create table if not exists table_name_id -( - table_name text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_name, table_namespace) -); - -create table if not exists table_path_id -( - table_path text, - table_id text, - table_namespace text default 'default', - domain text default 'public', - primary key (table_path) -); - -DO -$$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'data_file_op') THEN - create type data_file_op as - ( - path text, - file_op text, - size bigint, - file_exist_cols text - ); - END IF; - END -$$; - -create table if not exists data_commit_info -( - table_id text, - partition_desc text, - commit_id UUID, - file_ops data_file_op[], - commit_op text, - committed boolean default 'false', - timestamp bigint, - domain text default 'public', - primary key (table_id, partition_desc, commit_id) -); - -create table if not exists partition_info -( - table_id text, - partition_desc text, - version int, - commit_op text, - timestamp bigint DEFAULT (date_part('epoch'::text, now()) * (1000)::double precision), - snapshot UUID[], - expression text, - domain text default 'public', - primary key (table_id, partition_desc, version) -); - -CREATE OR REPLACE FUNCTION partition_insert() RETURNS TRIGGER AS -$$ -DECLARE - rs_version integer; - rs_table_path text; - rs_table_namespace text; -BEGIN - if NEW.commit_op <> 'CompactionCommit' then - select version - INTO rs_version - from partition_info - where table_id = NEW.table_id - and partition_desc = NEW.partition_desc - and version != NEW.version - and commit_op = 'CompactionCommit' - order by version desc - limit 1; - if rs_version >= 0 then - if NEW.version - rs_version >= 3 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - else - if NEW.version >= 2 then - select table_path, table_namespace - into rs_table_path, rs_table_namespace - from table_info - where table_id = NEW.table_id; - perform pg_notify('lakesoul_compaction_notify', - concat('{"table_path":"', rs_table_path, '","table_partition_desc":"', - NEW.partition_desc, '","table_namespace":"', rs_table_namespace, '"}')); - end if; - end if; - RETURN NULL; - end if; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER partition_table_change - AFTER INSERT - ON partition_info - FOR EACH ROW -EXECUTE PROCEDURE partition_insert(); - -create table if not exists global_config -( - key text, - value text, - primary key (key) -); - diff --git a/samples/datalake/lakesoul/sql/mysql_init.sql b/samples/datalake/lakesoul/sql/mysql_init.sql deleted file mode 100644 index d7fd3c01c5c817..00000000000000 --- a/samples/datalake/lakesoul/sql/mysql_init.sql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE TABLE customer ( - c_custkey INT NOT NULL, - c_name VARCHAR(25), - c_address VARCHAR(40), - c_nationkey INT, - c_phone CHAR(15), - c_acctbal DECIMAL(15,2), - c_mktsegment VARCHAR(10), - c_comment VARCHAR(117), - PRIMARY KEY (c_custkey) -); - -LOAD DATA INFILE '/data/tpch/customer.tbl' -INTO TABLE customer -FIELDS TERMINATED BY '|' -LINES TERMINATED BY '\n'; - -CREATE TABLE orders ( - o_orderkey INT NOT NULL, - o_custkey INT, - o_orderstatus CHAR(1), - o_totalprice DECIMAL(15,2), - o_orderdate DATE, - o_orderpriority CHAR(15), - o_clerk CHAR(15), - o_shippriority INT, - o_comment VARCHAR(79), - PRIMARY KEY (o_orderkey), - FOREIGN KEY (o_custkey) REFERENCES customer(c_custkey) -); - -LOAD DATA INFILE '/data/tpch/orders.tbl' -INTO TABLE orders -FIELDS TERMINATED BY '|' -LINES TERMINATED BY '\n'; \ No newline at end of file diff --git a/samples/datalake/lakesoul/sql/prepare_data.sql b/samples/datalake/lakesoul/sql/prepare_data.sql deleted file mode 100644 index a5b7de178b3f74..00000000000000 --- a/samples/datalake/lakesoul/sql/prepare_data.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE TABLE spark_catalog.default.customer ( - c_custkey INT NOT NULL, - c_name STRING, - c_address STRING, - c_nationkey INT, - c_phone STRING, - c_acctbal DECIMAL(15, 2), - c_mktsegment STRING, - c_comment STRING -) -USING csv -OPTIONS (path '/data/tpch/customer.tbl', delimiter '|'); - -USE lakesoul; -CREATE NAMESPACE IF NOT EXISTS tpch; - -CREATE TABLE IF NOT EXISTS tpch.customer_from_spark ( - c_custkey INT NOT NULL, - c_name STRING, - c_address STRING, - c_nationkey INT, - c_phone STRING, - c_acctbal DECIMAL(15, 2), - c_mktsegment STRING, - c_comment STRING) -USING lakesoul -PARTITIONED BY (c_nationkey) -LOCATION 's3://lakesoul-test-bucket/data/tpch/customer_from_spark' -TBLPROPERTIES ( - 'hashBucketNum' = '4', - 'hashPartitions' = 'c_custkey'); - - -INSERT OVERWRITE lakesoul.tpch.customer_from_spark -SELECT c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment, c_nationkey -FROM ( - SELECT COALESCE(c_custkey, -1) AS c_custkey, - c_name, c_address, c_phone, c_mktsegment, c_comment, c_acctbal, - COALESCE(CAST(c_nationkey AS int), -1) AS c_nationkey - FROM spark_catalog.default.customer -); diff --git a/samples/datalake/lakesoul/start_all.sh b/samples/datalake/lakesoul/start_all.sh deleted file mode 100644 index 760e4d659932df..00000000000000 --- a/samples/datalake/lakesoul/start_all.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# DORIS_PACKAGE=apache-doris-3.0.2-bin-x64 -# DORIS_DOWNLOAD_URL=https://apache-doris-releases.oss-accelerate.aliyuncs.com -DORIS_PACKAGE=apache-doris-3.0.2-lakesoul-bin-x64 -DORIS_DOWNLOAD_URL=https://lakesoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/doris -LAKESOUL_VERSION=2.6.1 -FLINK_LAKESOUL_JAR=lakesoul-flink-1.17-${LAKESOUL_VERSION}.jar -SPARK_LAKESOUL_JAR=lakesoul-spark-3.3-${LAKESOUL_VERSION}.jar - -download_source_file() { - local FILE_PATH="$1" - local EXPECTED_MD5="$2" - local DOWNLOAD_URL="$3" - - echo "solve for ${FILE_PATH} ..." - - if [[ -f "${FILE_PATH}" ]]; then - local FILE_MD5 - echo "compare md5sum ..." - FILE_MD5=$(md5sum "${FILE_PATH}" | awk '{ print $1 }') - - if [[ "${FILE_MD5}" = "${EXPECTED_MD5}" ]]; then - echo "${FILE_PATH} is ready!" - else - echo "${FILE_PATH} is broken, Redownloading ..." - rm "${FILE_PATH}" - wget "${DOWNLOAD_URL}"/"${FILE_PATH}" - fi - else - echo "downloading ${FILE_PATH} ...", - wget "${DOWNLOAD_URL}"/"${FILE_PATH}" - fi -} - -unpack_tar() { - local TAR_FILE_PATH="$1" - local OUT_DIR="$2" - - echo "unpack for ${TAR_FILE_PATH} ..." - - if [[ ! -f "${OUT_DIR}/SUCCESS" ]]; then - echo "Prepare ${OUT_DIR} environment" - if [[ -d "${OUT_DIR}" ]]; then - echo "Remove broken ${OUT_DIR}" - rm -rf ${OUT_DIR} - fi - echo "Unpackage ${TAR_FILE_PATH}" - tar xzf "${TAR_FILE_PATH}" - touch "${OUT_DIR}"/SUCCESS - fi -} - - -curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -cd "${curdir}" || exit - -if [[ ! -d "packages" ]]; then - mkdir packages -fi -cd packages || exit - -download_source_file "${DORIS_PACKAGE}.tar.gz" "b08d23cef21a34cee3a8ce337bed559f" "${DORIS_DOWNLOAD_URL}" - -download_source_file "${FLINK_LAKESOUL_JAR}" "de38f63ebd44835e9a37412908a0cdc0" "https://github.com/lakesoul-io/LakeSoul/releases/download/v${LAKESOUL_VERSION}" - -download_source_file "${SPARK_LAKESOUL_JAR}" "31a923958fb5398bbd00de073a88a787" "https://github.com/lakesoul-io/LakeSoul/releases/download/v${LAKESOUL_VERSION}" - -download_source_file "openjdk-17_linux-x64_bin.tar.gz" "5d1e9bc9be1570768485df4ff665821d" "https://mirrors.huaweicloud.com/openjdk/17/" -unpack_tar "openjdk-17_linux-x64_bin.tar.gz" "jdk-17" - -# download_source_file "hadoop-3.3.5.tar.gz" "1b6175712d813e8baec48ed68098ca85" "https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5" -download_source_file "hadoop-3.3.5.tar.gz" "1b6175712d813e8baec48ed68098ca85" "https://lakesoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/doris" -unpack_tar "hadoop-3.3.5.tar.gz" "hadoop-3.3.5" - - -download_source_file "flink-s3-fs-hadoop-1.17.1.jar" "0a631b07ba3e3b6c54e7c7c920ac6487" "https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.17.1" -download_source_file "parquet-hadoop-bundle-1.12.3.jar" "3a78d684a1938e68c6a57f59863e9106" "https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.12.3" -download_source_file "flink-parquet-1.17.1.jar" "559fda5535d4018fb923c4ec198340f0" "https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.17.1" - - -if [[ ! -f "doris-bin/SUCCESS" ]]; then - echo "Prepare ${DORIS_PACKAGE} environment" - if [[ -d "doris-bin" ]]; then - echo "Remove broken ${DORIS_PACKAGE}" - rm -rf doris-bin - fi - echo "Unpackage ${DORIS_PACKAGE}" - tar xzf "${DORIS_PACKAGE}".tar.gz - - mv "${DORIS_PACKAGE}" doris-bin - sed -i 's/-lt 2000000 ]]/-lt 60000 ]]/g' doris-bin/be/bin/start_be.sh - touch doris-bin/SUCCESS -fi - -if [[ ! -f "jars/SUCCESS" ]]; then - echo "Prepare jars environment" - if [[ -d "jars" ]]; then - echo "Remove broken jars" - rm -rf jars - fi - mkdir jars - cp ./*.jar jars/ - touch jars/SUCCESS -fi - -if [[ ! -f "tpch-dbgen/SUCCESS" ]]; then - echo "Load tpch data..." - if [[ -d "tpch-dbgen" ]]; then - echo "Remove broken tpch-dbgen" - rm -rf tpch-dbgen - fi - - git clone https://github.com/databricks/tpch-dbgen.git - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - - touch SUCCESS - cd ../ -fi - -cd ../ - -echo "Start docker-compose..." -sudo docker compose -f docker-compose.yml --env-file docker-compose.env up -d - -echo "Start prepare data for lakesoul tables..." -sudo docker exec -it doris-lakesoul-spark spark-sql --conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem --conf spark.hadoop.fs.s3a.buffer.dir=/opt/spark/work-dir/s3a --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider --conf spark.hadoop.fs.s3a.access.key=admin --conf spark.hadoop.fs.s3a.secret.key=password -f /opt/sql/prepare_data.sql | tee -a init.log >/dev/null - - -echo "=============================================================================" -echo "Success to launch doris+iceberg+paimon+flink+spark+minio environments!" -echo "You can:" -echo " 'bash start_doris_client.sh' to login into doris" -echo " 'bash start_flink_client.sh' to login into flink" -echo " 'bash start_spark_sql.sh' to login into spark" -echo " 'bash start_mysql_client.sh' to login into mysql for test" -echo "=============================================================================" diff --git a/samples/datalake/lakesoul/start_doris_client.sh b/samples/datalake/lakesoul/start_doris_client.sh deleted file mode 100644 index a411102b7978ed..00000000000000 --- a/samples/datalake/lakesoul/start_doris_client.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-doris sh -c "mysql -u root -h doris -P 9030" diff --git a/samples/datalake/lakesoul/start_flink_cdc_job.sh b/samples/datalake/lakesoul/start_flink_cdc_job.sh deleted file mode 100644 index cd20f1f1de7ffb..00000000000000 --- a/samples/datalake/lakesoul/start_flink_cdc_job.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -echo "Start flink-cdc job..." -sudo docker exec -t doris-lakesoul-jobmanager flink run -d -c org.apache.flink.lakesoul.entry.MysqlCdc /opt/flink/jars/lakesoul-flink-1.17-2.6.1.jar --source_db.host mysql --source_db.port 3306 --source_db.db_name tpch --source_db.user root --source_db.password root --source.parallelism 2 --sink.parallelism 4 --use.cdc true --warehouse_path s3://lakesoul-test-bucket/data/ --flink.checkpoint s3://lakesoul-test-bucket/chk --flink.savepoint s3://lakesoul-test-bucket/svp --job.checkpoint_interval 5000 --server_time_zone UTC diff --git a/samples/datalake/lakesoul/start_flink_client.sh b/samples/datalake/lakesoul/start_flink_client.sh deleted file mode 100644 index e30634b5f89dfc..00000000000000 --- a/samples/datalake/lakesoul/start_flink_client.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-jobmanager sql-client.sh diff --git a/samples/datalake/lakesoul/start_mysql_client.sh b/samples/datalake/lakesoul/start_mysql_client.sh deleted file mode 100644 index e298343f488ac2..00000000000000 --- a/samples/datalake/lakesoul/start_mysql_client.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker exec -it doris-lakesoul-mysql-source sh -c "mysql -u root -h mysql -p tpch -P 3306" diff --git a/samples/datalake/lakesoul/start_spark_sql.sh b/samples/datalake/lakesoul/start_spark_sql.sh deleted file mode 100644 index d6d777923f6f38..00000000000000 --- a/samples/datalake/lakesoul/start_spark_sql.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -docker exec -it doris-lakesoul-spark spark-sql --conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem --conf spark.hadoop.fs.s3a.buffer.dir=/opt/spark/work-dir/s3a --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider --conf spark.hadoop.fs.s3a.access.key=admin --conf spark.hadoop.fs.s3a.secret.key=password \ No newline at end of file diff --git a/samples/datalake/lakesoul/stop_all.sh b/samples/datalake/lakesoul/stop_all.sh deleted file mode 100644 index 63c244756078da..00000000000000 --- a/samples/datalake/lakesoul/stop_all.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo docker compose -f docker-compose.yml --env-file docker-compose.env down diff --git a/samples/datalake/lakesoul/work-dir/lakesoul.properties b/samples/datalake/lakesoul/work-dir/lakesoul.properties deleted file mode 100644 index a37fffa1ae40ef..00000000000000 --- a/samples/datalake/lakesoul/work-dir/lakesoul.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -lakesoul.pg.driver=com.lakesoul.shaded.org.postgresql.Driver -lakesoul.pg.url=jdbc:postgresql://doris-lakesoul-docker-compose-env-lakesoul-meta-db-1:5432/lakesoul_test?stringtype=unspecified -lakesoul.pg.username=lakesoul_test -lakesoul.pg.password=lakesoul_test