From 78fb48e145cdc6bc4548b4614106da240b98b8fd Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 8 Sep 2026 12:49:57 +0800 Subject: [PATCH] [fix](fe) Keep rollup jobs waiting on conflict txn abort failure ### What problem does this PR solve? Issue Number: None Related PR: #65196 Problem Summary: A rollup job in WAITING_TXN checks and aborts failed conflict transactions before proceeding. If a transaction is already committed, becomes visible, or is cleaned up before the abort completes, abortTransaction() can throw UserException. RollupJobV2 propagates this as AlterCancelException, causing the entire rollup job to be cancelled. Restarting FE with committed but not yet visible transactions reproduces this failure. Apply the retry behavior introduced for schema change in #65196 to rollup jobs: log the abort failure, keep the job in WAITING_TXN, and re-query transaction state in the next scheduler round. Also exclude committed and final transactions from the shared failed-transaction selector to avoid unnecessary abort attempts. The exception handling remains necessary because transaction state can change after selection. ### Release note Prevent rollup and synchronous materialized view jobs from being cancelled when aborting a conflicting transaction fails. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - run-fe-ut.sh: GlobalTransactionMgrTest#testCheckFailedTxnsWithOfflineCoordinator and RollupJobV2Test#testCommitWhileAbortingPreviousLoad; 2 tests passed - run-regression-test.sh: insert_p0/transaction, txn_insert_restart_fe_with_schema_change, not_cloud mode; 1 suite passed, 0 skipped - BUILD_TYPE=ASAN ./build.sh --fe -j8 passed - FE Core Checkstyle: 0 violations; git diff --check passed - Behavior changed: Yes; rollup jobs wait and retry when conflict transaction abort fails, and committed or final transactions are excluded from automatic abort selection - Does this need documentation: No --- .../org/apache/doris/alter/RollupJobV2.java | 10 ++++- .../transaction/GlobalTransactionMgr.java | 4 ++ .../apache/doris/alter/RollupJobV2Test.java | 37 +++++++++++++++++++ .../transaction/GlobalTransactionMgrTest.java | 19 ++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 7c5523373b4218..060e24e3be4815 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -708,8 +708,14 @@ protected boolean checkFailedPreviousLoadAndAbort() throws UserException { if (Config.enable_abort_txn_by_checking_conflict_txn) { List failedTxns = GlobalTransactionMgr.checkFailedTxns(unFinishedTxns); for (TransactionState txn : failedTxns) { - Env.getCurrentGlobalTransactionMgr() - .abortTransaction(txn.getDbId(), txn.getTransactionId(), "Cancel by schema change"); + try { + Env.getCurrentGlobalTransactionMgr() + .abortTransaction(txn.getDbId(), txn.getTransactionId(), "Cancel by schema change"); + } catch (UserException e) { + LOG.warn("failed to abort previous load txn {}, wait next round. rollup job: {}", + txn.getTransactionId(), jobId, e); + return false; + } } } return unFinishedTxns.isEmpty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index 3d04668e2a651f..89b2f415782a08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -515,6 +515,10 @@ public static boolean checkFailedTxnsByCoordinator(TransactionState txn) { public static List checkFailedTxns(List conflictTxns) { List failedTxns = new ArrayList<>(); for (TransactionState txn : conflictTxns) { + TransactionStatus status = txn.getTransactionStatus(); + if (status == TransactionStatus.COMMITTED || status.isFinalStatus()) { + continue; + } if (checkFailedTxnsByCoordinator(txn)) { failedTxns.add(txn); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index c525ea2d4a48cd..b796516ab67aee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -55,7 +55,13 @@ import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TTaskType; import org.apache.doris.transaction.FakeTransactionIDGenerator; +import org.apache.doris.transaction.GlobalTransactionMgr; import org.apache.doris.transaction.GlobalTransactionMgrIface; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TransactionState.LoadJobSourceType; +import org.apache.doris.transaction.TransactionState.TxnCoordinator; +import org.apache.doris.transaction.TransactionState.TxnSourceType; +import org.apache.doris.transaction.TransactionStatus; import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterEach; @@ -144,6 +150,37 @@ public void tearDown() { } } + @Test + public void testCommitWhileAbortingPreviousLoad() throws Exception { + long txnId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, + Lists.newArrayList(CatalogTestUtil.testTableId1), "commit_during_rollup_abort", + new TxnCoordinator(TxnSourceType.FE, 0, "missing", 0), LoadJobSourceType.FRONTEND, 60); + TransactionState txn = masterTransMgr.getTransactionState(CatalogTestUtil.testDbId1, txnId); + Database db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + MaterializedViewHandler handler = masterEnv.getMaterializedViewHandler(); + handler.process(Lists.newArrayList(op), db, table); + RollupJobV2 job = (RollupJobV2) handler.getAlterJobsV2().values().iterator().next(); + job.jobState = JobState.WAITING_TXN; + job.watershedTxnId = txnId + 1; + + try (MockedStatic mocked = Mockito.mockStatic( + GlobalTransactionMgr.class, Mockito.CALLS_REAL_METHODS)) { + mocked.when(() -> GlobalTransactionMgr.checkFailedTxns(Mockito.anyList())).thenAnswer(invocation -> { + List failed = (List) invocation.callRealMethod(); + Assertions.assertEquals(List.of(txn), failed); + txn.setTransactionStatus(TransactionStatus.COMMITTED); + return failed; + }); + job.runWaitingTxnJob(); + Assertions.assertEquals(JobState.WAITING_TXN, job.getJobState()); + Assertions.assertEquals(TransactionStatus.COMMITTED, txn.getTransactionStatus()); + } + Assertions.assertFalse(job.checkFailedPreviousLoadAndAbort()); + txn.setTransactionStatus(TransactionStatus.VISIBLE); + Assertions.assertTrue(job.checkFailedPreviousLoadAndAbort()); + } + @Test public void testRunRollupJobConcurrentLimit() throws UserException { if (fakeEnv != null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java index b76370ea5e4365..0231a6360e8e0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java @@ -126,6 +126,25 @@ public void tearDown() { } } + @Test + public void testCheckFailedTxnsWithOfflineCoordinator() { + FakeEnv.setEnv(masterEnv); + for (TxnSourceType source : List.of(TxnSourceType.FE, TxnSourceType.BE)) { + TransactionState txn = new TransactionState(CatalogTestUtil.testDbId1, + Lists.newArrayList(CatalogTestUtil.testTableId1), 1, "offline_coordinator", null, + LoadJobSourceType.FRONTEND, new TxnCoordinator(source, Long.MAX_VALUE, "missing", 0), -1, 60000); + for (TransactionStatus status : List.of(TransactionStatus.PREPARE, TransactionStatus.PRECOMMITTED)) { + txn.setTransactionStatus(status); + Assertions.assertEquals(List.of(txn), GlobalTransactionMgr.checkFailedTxns(List.of(txn))); + } + for (TransactionStatus status : List.of(TransactionStatus.COMMITTED, + TransactionStatus.VISIBLE, TransactionStatus.ABORTED)) { + txn.setTransactionStatus(status); + Assertions.assertTrue(GlobalTransactionMgr.checkFailedTxns(List.of(txn)).isEmpty()); + } + } + } + @Test public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisException, BeginTransactionException, DuplicatedRequestException, QuotaExceedException, MetaNotFoundException {