From 3d05566e61369f55235b6152732f3fa63eb92f92 Mon Sep 17 00:00:00 2001 From: Yixuan Wang Date: Tue, 17 Nov 2026 01:36:20 -0800 Subject: [PATCH] [fix](cloud) Fix infinite loop and OOM in FDB conflict logging (#70417) ### Background When an FDB commit fails with a transaction conflict, Meta Service can log the conflicting keys and the transaction's read and write conflict ranges. These diagnostics read FDB special key ranges when enable_logging_conflict_keys is enabled. ### What problem does this PR solve? Issue Number: None Related PR: None PR Type: Bugfix Problem Summary: Conflict logging can enter an infinite loop when a diagnostic range spans multiple pages. The loop repeatedly reads the same completed FDB future and appends the same key/value pairs, causing unbounded memory growth and potentially an MS OOM. Read all three diagnostic ranges through the existing full-range iterator so each page advances on the same transaction. Keep full keys for paging and strip the special key prefix only from copied results. ### Details - **Frequency:** Triggered when conflict logging is enabled, an FDB commit conflicts, and a diagnostic range returns more=true. - **Root cause:** Each getter creates one range-read future outside its loop. fdb_future_get_keyvalue_array only extracts that future's result; it does not fetch the next page. Repeating it leaves more=true and keeps copying the same page into the result vector. - **Why existing tests missed it:** The existing test uses one conflicting key and reads both range ends in one page. - **Fix:** Share one helper backed by full_range_get across the conflicting-key, read-conflict, and write-conflict getters. Reuse its full-key cursor and future ownership, copy each result before advancing, and return iterator errors. The original commit error remains unchanged. - **Other paths and branches:** All three conflict diagnostic getters use the same fix. No backports are included. - **Behavior change:** Paginated conflict diagnostics finish reading their results or return a read error instead of looping over one page until memory is exhausted. ### Testing - Extended TxnKvTest.ReportConflictingRange to force one key/value pair per page for all three getters and compare the complete results with a single-page read. The test also covers an embedded zero byte in the key and a timeout on the second page. - Static review only. The updated tests have not been run. Compilation and formatting checks were not run, as requested by the author. ### Release note Fix a possible Meta Service OOM caused by an infinite loop while logging paginated FDB conflict ranges with enable_logging_conflict_keys enabled. ### Check List (For Author) - Behavior changed: Yes (see Details) - Does this need documentation: No - Pre-submission status: Partially verified: static review only; compilation, formatting checks, and unit tests have not been run. ### AI Disclosure - Tool: Codex (GPT-6) - Generated: Code, test changes, and this description. - Human review: Not confirmed. ### Background ### What problem does this PR solve? Issue Number: Related PR: PR Type: Problem Summary: ### Details ### Testing ### Notes for Reviewers ### Release note ### Check List (For Author) - Behavior changed: - Does this need documentation: - Pre-submission status: ### AI Disclosure - Tool: - Generated: - Human review: --- cloud/src/meta-store/txn_kv.cpp | 142 +++++--------------------------- cloud/test/txn_kv_test.cpp | 60 +++++++++++++- 2 files changed, 80 insertions(+), 122 deletions(-) diff --git a/cloud/src/meta-store/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp index 2f2b04585807cb..130e6dd0db65c2 100644 --- a/cloud/src/meta-store/txn_kv.cpp +++ b/cloud/src/meta-store/txn_kv.cpp @@ -1040,136 +1040,38 @@ TxnErrorCode Transaction::get_versionstamp(Versionstamp* versionstamp) { return TxnErrorCode::TXN_OK; } +static TxnErrorCode get_conflict_range(Transaction* txn, std::string_view prefix, + std::vector>* values) { + std::string end(prefix); + end.push_back('\xff'); + FullRangeGetOptions opts; + opts.batch_limit = 0; + TEST_SYNC_POINT_CALLBACK("transaction:get_conflict_range:batch_limit", &opts.batch_limit); + + // The iterator uses the full last key to read the next page in the same transaction. + auto iter = txn->full_range_get(prefix, end, opts); + for (auto kv = iter->next(); kv.has_value(); kv = iter->next()) { + auto [key, value] = *kv; + // Drop the prefix only from the key copied to the result. + key.remove_prefix(prefix.size()); + values->emplace_back(key, value); + } + return iter->error_code(); +} + TxnErrorCode Transaction::get_conflicting_range( std::vector>* values) { - constexpr std::string_view start = "\xff\xff/transaction/conflicting_keys/"; - constexpr std::string_view end = "\xff\xff/transaction/conflicting_keys/\xff"; - - int limit = 0; - int target_bytes = 0; - FDBStreamingMode mode = FDB_STREAMING_MODE_WANT_ALL; - int iteration = 0; - fdb_bool_t snapshot = 0; - fdb_bool_t reverse = 0; - FDBFuture* future = fdb_transaction_get_range( - txn_, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)start.data(), start.size()), - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)end.data(), end.size()), limit, - target_bytes, mode, iteration, snapshot, reverse); - - DORIS_CLOUD_DEFER { - fdb_future_destroy(future); - }; - - RETURN_IF_ERROR(await_future(future)); - - FDBKeyValue const* out_kvs; - int out_kvs_count; - fdb_bool_t out_more = false; - do { - fdb_error_t err = - fdb_future_get_keyvalue_array(future, &out_kvs, &out_kvs_count, &out_more); - if (err) { - LOG(WARNING) << "get_conflicting_range get keyvalue array error: " - << fdb_get_error(err); - return cast_as_txn_code(err); - } - for (int i = 0; i < out_kvs_count; i++) { - std::string_view key((char*)out_kvs[i].key, out_kvs[i].key_length); - std::string_view value((char*)out_kvs[i].value, out_kvs[i].value_length); - key.remove_prefix(start.size()); - values->emplace_back(key, value); - } - } while (out_more); - - return TxnErrorCode::TXN_OK; + return get_conflict_range(this, "\xff\xff/transaction/conflicting_keys/", values); } TxnErrorCode Transaction::get_read_conflict_range( std::vector>* values) { - constexpr std::string_view start = "\xff\xff/transaction/read_conflict_range/"; - constexpr std::string_view end = "\xff\xff/transaction/read_conflict_range/\xff"; - - int limit = 0; - int target_bytes = 0; - FDBStreamingMode mode = FDB_STREAMING_MODE_WANT_ALL; - int iteration = 0; - fdb_bool_t snapshot = 0; - fdb_bool_t reverse = 0; - FDBFuture* future = fdb_transaction_get_range( - txn_, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)start.data(), start.size()), - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)end.data(), end.size()), limit, - target_bytes, mode, iteration, snapshot, reverse); - - DORIS_CLOUD_DEFER { - fdb_future_destroy(future); - }; - - RETURN_IF_ERROR(await_future(future)); - - FDBKeyValue const* out_kvs; - int out_kvs_count; - fdb_bool_t out_more; - do { - fdb_error_t err = - fdb_future_get_keyvalue_array(future, &out_kvs, &out_kvs_count, &out_more); - if (err) { - LOG(WARNING) << "get_conflicting_range get keyvalue array error: " - << fdb_get_error(err); - return cast_as_txn_code(err); - } - for (int i = 0; i < out_kvs_count; i++) { - std::string_view key((char*)out_kvs[i].key, out_kvs[i].key_length); - std::string_view value((char*)out_kvs[i].value, out_kvs[i].value_length); - key.remove_prefix(start.size()); - values->emplace_back(key, value); - } - } while (out_more); - - return TxnErrorCode::TXN_OK; + return get_conflict_range(this, "\xff\xff/transaction/read_conflict_range/", values); } TxnErrorCode Transaction::get_write_conflict_range( std::vector>* values) { - constexpr std::string_view start = "\xff\xff/transaction/write_conflict_range/"; - constexpr std::string_view end = "\xff\xff/transaction/write_conflict_range/\xff"; - - int limit = 0; - int target_bytes = 0; - FDBStreamingMode mode = FDB_STREAMING_MODE_WANT_ALL; - int iteration = 0; - fdb_bool_t snapshot = 0; - fdb_bool_t reverse = 0; - FDBFuture* future = fdb_transaction_get_range( - txn_, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)start.data(), start.size()), - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)end.data(), end.size()), limit, - target_bytes, mode, iteration, snapshot, reverse); - - DORIS_CLOUD_DEFER { - fdb_future_destroy(future); - }; - - RETURN_IF_ERROR(await_future(future)); - - FDBKeyValue const* out_kvs; - int out_kvs_count; - fdb_bool_t out_more; - do { - fdb_error_t err = - fdb_future_get_keyvalue_array(future, &out_kvs, &out_kvs_count, &out_more); - if (err) { - LOG(WARNING) << "get_conflicting_range get keyvalue array error: " - << fdb_get_error(err); - return cast_as_txn_code(err); - } - for (int i = 0; i < out_kvs_count; i++) { - std::string_view key((char*)out_kvs[i].key, out_kvs[i].key_length); - std::string_view value((char*)out_kvs[i].value, out_kvs[i].value_length); - key.remove_prefix(start.size()); - values->emplace_back(key, value); - } - } while (out_more); - - return TxnErrorCode::TXN_OK; + return get_conflict_range(this, "\xff\xff/transaction/write_conflict_range/", values); } TxnErrorCode Transaction::report_conflicting_range() { diff --git a/cloud/test/txn_kv_test.cpp b/cloud/test/txn_kv_test.cpp index f2771a502cf6bf..6f82d5b86ff309 100644 --- a/cloud/test/txn_kv_test.cpp +++ b/cloud/test/txn_kv_test.cpp @@ -1547,10 +1547,16 @@ TEST(TxnKvTest, BatchScan) { } TEST(TxnKvTest, ReportConflictingRange) { + const bool old_enable_logging_conflict_keys = config::enable_logging_conflict_keys; + DORIS_CLOUD_DEFER { + config::enable_logging_conflict_keys = old_enable_logging_conflict_keys; + }; config::enable_logging_conflict_keys = true; constexpr std::string_view key_prefix = "txn_kv_test__report_conflicting_range"; std::string key = std::string(key_prefix) + std::to_string(time(nullptr)); + key.push_back('\0'); + key.append("suffix"); { // 1. write a common key @@ -1576,13 +1582,63 @@ TEST(TxnKvTest, ReportConflictingRange) { ASSERT_EQ(txn2->commit(), TxnErrorCode::TXN_CONFLICT); // 3. get the conflicting ranges. + auto* fdb_txn = static_cast(txn2.get()); std::vector> values; - ASSERT_EQ(reinterpret_cast(txn2.get())->get_conflicting_range(&values), - TxnErrorCode::TXN_OK); + ASSERT_EQ(fdb_txn->get_conflicting_range(&values), TxnErrorCode::TXN_OK); ASSERT_EQ(values.size(), 2); ASSERT_EQ(values[0].first, key); ASSERT_EQ(values[1].second, "0"); ASSERT_TRUE(values[1].first.starts_with(key)); + + auto* sp = doris::SyncPoint::get_instance(); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + sp->clear_all_call_backs(); + sp->disable_processing(); + }; + + int batch_limit = 0; + doris::SyncPoint::CallbackGuard limit_guard; + sp->set_call_back( + "transaction:get_conflict_range:batch_limit", + [&](auto&& args) { *doris::try_any_cast(args[0]) = batch_limit; }, &limit_guard); + int page_count = 0; + int fail_on_page = 0; + constexpr fdb_error_t txn_timed_out = 1031; + doris::SyncPoint::CallbackGuard page_guard; + sp->set_call_back( + "range_get_iterator:init:get_keyvalue_array_err", + [&](auto&& args) { + if (++page_count == fail_on_page) { + *doris::try_any_cast(args[0]) = txn_timed_out; + } + }, + &page_guard); + + for (auto reader : + {&fdb::Transaction::get_conflicting_range, &fdb::Transaction::get_read_conflict_range, + &fdb::Transaction::get_write_conflict_range}) { + batch_limit = 0; + fail_on_page = 0; + values.clear(); + ASSERT_EQ((fdb_txn->*reader)(&values), TxnErrorCode::TXN_OK); + ASSERT_EQ(values.size(), 2); + const auto expected = values; + + // Put the two range ends on separate pages. + batch_limit = 1; + page_count = 0; + values.clear(); + ASSERT_EQ((fdb_txn->*reader)(&values), TxnErrorCode::TXN_OK); + EXPECT_EQ(values, expected); + EXPECT_GT(page_count, 1); + + page_count = 0; + fail_on_page = 2; + values.clear(); + EXPECT_EQ((fdb_txn->*reader)(&values), TxnErrorCode::TXN_TIMEOUT); + EXPECT_EQ(values.size(), 1); + } } TEST(TxnKvTest, WatchKey) {