diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 4d708208c5b3df..11f50909ccc269 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; @@ -625,6 +626,14 @@ private static MTMVAnalyzeQueryInfo analyzeQueryInternal(ConnectContext ctx, Map properties = CreateTableInfo.addOlapHiddenColumns( columns, isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS, isIvm, properties, false); + // A row-binlog table carries hidden columns on top of the OLAP ones above, added by + // InternalCatalog#createOlapTable just before the table is built. The analyzed list has + // to carry them too: an MTMV re-validates its schema against it whenever a base table + // changes (MTMVPlanUtil#checkColumnIfChange), and a column missing here is + // indistinguishable from a real schema change. Idempotent, so the table still gets one. + CreateTableInfo.addRowBinlogHiddenColumns(columns, + isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS, isIvm, + BinlogConfig.fromProperties(properties)); // analyze column final boolean finalEnableMergeOnWrite = isIvm; Set keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 69357d6f39a405..6c5e728e84f7d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -1646,14 +1646,34 @@ public void checkLegalityOfPartitionExprs(PartitionTableInfo partitionTableInfo) * Add hidden columns required by row binlog. */ public void createRowBinlogHiddenColumnsIfNecessary(BinlogConfig binlogConfig) { - if (!binlogConfig.isRowFormat()) { + addRowBinlogHiddenColumns(columns, keysType, isEnableMergeOnWrite, binlogConfig); + } + + /** + * Append the hidden columns a row-binlog table carries. Callers that build the column list + * outside the create-table flow (an analyzed MTMV schema) go through here as well, so both + * sides of {@code MTMVPlanUtil#checkColumnIfChange} agree on the physical layout. + * + *

Idempotent: a column that is already present is kept once. + */ + public static void addRowBinlogHiddenColumns(List columns, KeysType keysType, + boolean isEnableMergeOnWrite, BinlogConfig binlogConfig) { + if (binlogConfig == null || !binlogConfig.isRowFormat()) { return; } if (keysType.equals(KeysType.DUP_KEYS)) { - columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE)); - columns.add(ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE)); + addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE)); + addIfAbsent(columns, ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE)); } else if (keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite) { - columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE)); + addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE)); + } + } + + private static void addIfAbsent(List columns, ColumnDefinition columnDefinition) { + boolean present = columns.stream() + .anyMatch(column -> column.getName().equalsIgnoreCase(columnDefinition.getName())); + if (!present) { + columns.add(columnDefinition); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java index 162660ed8e5088..c2b0f83d44f6bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java @@ -490,6 +490,70 @@ public void testEnsureMTMVQueryUsableEnableIvmRewriteByRefreshMethod() throws Ex incrementalCtx.getStatementContext().getIvmRewriteContext().orElseThrow().getMode()); } + @Test + public void testEnsureMTMVQueryUsableWithRowBinlogHiddenColumns() throws Exception { + // A table created with row binlog carries hidden columns its query never produces: + // __DORIS_COMMIT_TSO_COL__ for merge-on-write unique keys, plus __DORIS_ROW_LSN_COL__ for + // duplicate keys. InternalCatalog#createOlapTable adds them while the table is built, so an + // MV that enables row binlog itself ends up with them in its physical schema. + // ensureMTMVQueryUsable re-derives the schema from the query and compares the two + // (checkColumnIfChange), so the analyzed column list has to carry them too -- otherwise + // every refresh of such an MV fails with a spurious "column length not equals". + createTable("CREATE TABLE IF NOT EXISTS row_binlog_schema_base (\n" + + " k1 int,\n" + + " v1 int\n" + + ")\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')\n"); + + createMvByNereids("create materialized view row_binlog_schema_ivm " + + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', " + + "'binlog.format' = 'ROW') \n" + + " as select k1, v1 from test.row_binlog_schema_base;"); + createMvByNereids("create materialized view row_binlog_schema_dup " + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', " + + "'binlog.format' = 'ROW') \n" + + " as select k1, v1 from test.row_binlog_schema_base;"); + createMvByNereids("create materialized view row_binlog_schema_plain " + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select k1, v1 from test.row_binlog_schema_base;"); + + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV ivmMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_ivm"); + MTMV dupMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_dup"); + MTMV plainMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_plain"); + + Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL), rowBinlogHiddenColumns(ivmMv)); + Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL, Column.ROW_LSN_COL), + rowBinlogHiddenColumns(dupMv)); + Assertions.assertTrue(rowBinlogHiddenColumns(plainMv).isEmpty()); + + for (MTMV mtmv : Lists.newArrayList(ivmMv, dupMv, plainMv)) { + ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv, + MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE); + Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, ctx), + "analyzed schema must match the physical schema of " + mtmv.getName()); + } + } + + private static List rowBinlogHiddenColumns(MTMV mtmv) { + List hidden = Lists.newArrayList(); + for (Column column : mtmv.getBaseSchema(true)) { + if (column.getName().equalsIgnoreCase(Column.COMMIT_TSO_COL) + || column.getName().equalsIgnoreCase(Column.ROW_LSN_COL)) { + hidden.add(column.getName().toUpperCase()); + } + } + return hidden; + } + @Test public void testEnsureMTMVQueryAnalyzeFailed() throws Exception { createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n" diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.out b/regression-test/data/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.out new file mode 100644 index 00000000000000..9d9acb797cb479 --- /dev/null +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rb_ivm_rows -- +1 10 +2 20 + +-- !rb_ivm_no_binlog_rows -- +1 10 +2 20 + +-- !rb_dup_rows -- +1 10 +2 20 + +-- !rb_dup_no_binlog_rows -- +1 10 +2 20 + +-- !cascade_l2_baseline -- +10 2 300 +20 1 300 + +-- !cascade_l1_after_aba -- +0 3 600 + +-- !cascade_l2_after_aba -- +0 3 600 + diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.groovy new file mode 100644 index 00000000000000..b93dff85fddcf4 --- /dev/null +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_row_binlog_schema_validation.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.SECONDS + +// A materialized view re-validates its schema against a fresh analysis of its own query whenever a +// base table changes (MTMVPlanUtil.ensureMTMVQueryUsable -> checkColumnIfChange). A table created +// with row binlog carries hidden columns that the query never produces, and an MV carries them too +// when its own properties enable row binlog (which cascade IVM requires). The analyzed schema has to +// contain them as well, otherwise every refresh of such an MV fails with +// "column length not equals, please check whether columns of base table have changed" -- including a +// COMPLETE refresh, which is the only way out of a stale baseline. +// +// Covered here: the four MV shapes, and the DORIS-28306 shape where a cascade L1 could not recover +// through COMPLETE after a schema ABA on a referenced column. +suite("test_ivm_row_binlog_schema_validation") { + def rowBinlogProps = "'replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW'" + def plainProps = "'replication_num' = '1'" + // A cascade source keeps historical values so its own downstream MV can read the binlog. + def cascadeProps = rowBinlogProps + ", 'binlog.need_historical_value' = 'true'" + + def ddlJobCount = { String table -> + return sql("""SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'""").size() + } + + // `SHOW ALTER TABLE COLUMN` keeps finished jobs, so wait for a *new* job that is FINISHED. + def waitDdlFinished = { String table, int previousJobCount -> + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def jobs = sql """SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'""" + return jobs.size() > previousJobCount + && jobs.every({ row -> row[9].toString() == 'FINISHED' }) + }) + } + + def lastTaskId = null + // tasks('type'='mv') can briefly miss the just-finished task, so require a *new* TaskId. + def waitTerminalTask = { String mv -> + def taskResult + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + taskResult = sql_return_maparray(""" + SELECT TaskId, Status, RefreshMode, ErrorMsg + FROM tasks('type'='mv') + WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mv}' + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 + """) + return !taskResult.isEmpty() + && taskResult[0].TaskId.toString() != lastTaskId + && taskResult[0].Status.toString() != 'PENDING' + && taskResult[0].Status.toString() != 'RUNNING' + }) + lastTaskId = taskResult[0].TaskId.toString() + return taskResult[0] + } + + def refresh = { String mv, String mode -> + sql """REFRESH MATERIALIZED VIEW ${mv} ${mode}""" + return waitTerminalTask(mv) + } + + // ---------------------------------------------------------------- 1. four MV shapes + // Same query and same base table for all four; what differs is whether the MV itself enables + // row binlog, and (for the IVM ones) whether the MV is a MOW unique table. + def cases = [ + [name: "rb_ivm", mvProps: rowBinlogProps, ivm: true], + [name: "rb_ivm_no_binlog", mvProps: plainProps, ivm: true], + [name: "rb_dup", mvProps: rowBinlogProps, ivm: false], + [name: "rb_dup_no_binlog", mvProps: plainProps, ivm: false], + ] + + for (def c : cases) { + def table = c.name + "_base" + def mv = c.name + "_mv" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv}""" + sql """DROP TABLE IF EXISTS ${table}""" + sql """ + CREATE TABLE ${table} ( + k1 INT NOT NULL, + v1 INT NULL, + spare INT NULL + ) + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true', + 'binlog.enable' = 'true', 'binlog.format' = 'ROW', + 'binlog.need_historical_value' = 'true') + """ + sql """INSERT INTO ${table} VALUES (1, 10, 1), (2, 20, 1)""" + + sql """ + CREATE MATERIALIZED VIEW ${mv} + BUILD DEFERRED REFRESH ${c.ivm ? 'INCREMENTAL' : 'COMPLETE'} ON MANUAL + KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES (${c.mvProps}) + AS SELECT k1, SUM(v1) AS total FROM ${table} GROUP BY k1 + """ + def task = refresh(mv, "COMPLETE") + assertEquals("SUCCESS", task.Status.toString(), "${c.name} baseline refresh: " + task.ErrorMsg) + + // A schema change that leaves the query intact: the MV must stay refreshable. The dropped + // column is not referenced, so nothing about the MV's result changes. + def before = ddlJobCount(table) + sql """ALTER TABLE ${table} DROP COLUMN spare""" + waitDdlFinished(table, before) + + task = refresh(mv, c.ivm ? "INCREMENTAL" : "COMPLETE") + assertEquals("SUCCESS", task.Status.toString(), + "${c.name} refresh after an unrelated DROP COLUMN: " + task.ErrorMsg) + } + + // The dropped column was never referenced, so all four MVs must still hold their baseline rows. + order_qt_rb_ivm_rows "SELECT k1, total FROM rb_ivm_mv" + order_qt_rb_ivm_no_binlog_rows "SELECT k1, total FROM rb_ivm_no_binlog_mv" + order_qt_rb_dup_rows "SELECT k1, total FROM rb_dup_mv" + order_qt_rb_dup_no_binlog_rows "SELECT k1, total FROM rb_dup_no_binlog_mv" + + // ------------------------------------- 2. DORIS-28306: cascade L1 must recover via COMPLETE + // A cascade forces row binlog onto L1, so L1 is exactly the shape that used to make every + // refresh of L1 fail after a schema change. An ABA on a referenced column must still leave the + // explicit COMPLETE recovery path usable. + def cTable = "cascade_aba_base" + sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l2""" + sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l1""" + sql """DROP TABLE IF EXISTS ${cTable}""" + sql """ + CREATE TABLE ${cTable} ( + id BIGINT NOT NULL, + grp INT NULL, + amount BIGINT NULL + ) + UNIQUE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true', + 'binlog.enable' = 'true', 'binlog.format' = 'ROW', + 'binlog.need_historical_value' = 'true') + """ + sql """INSERT INTO ${cTable} VALUES (1, 10, 100), (2, 10, 200), (3, 20, 300)""" + + sql """ + CREATE MATERIALIZED VIEW cascade_l1 + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(id) + PROPERTIES (${cascadeProps}) + AS SELECT id, grp, amount FROM ${cTable} + """ + def cTask = refresh("cascade_l1", "COMPLETE") + assertEquals("SUCCESS", cTask.Status.toString(), "cascade L1 baseline: " + cTask.ErrorMsg) + + sql """ + CREATE MATERIALIZED VIEW cascade_l2 + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(grp) + PROPERTIES (${cascadeProps}) + AS SELECT grp, COUNT(*) AS row_count, SUM(amount) AS total_amount FROM cascade_l1 GROUP BY grp + """ + cTask = refresh("cascade_l2", "COMPLETE") + assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 baseline: " + cTask.ErrorMsg) + order_qt_cascade_l2_baseline "SELECT grp, row_count, total_amount FROM cascade_l2" + + def cBefore = ddlJobCount(cTable) + sql """ALTER TABLE ${cTable} DROP COLUMN grp""" + waitDdlFinished(cTable, cBefore) + cBefore = ddlJobCount(cTable) + sql """ALTER TABLE ${cTable} ADD COLUMN grp INT NULL DEFAULT '0'""" + waitDdlFinished(cTable, cBefore) + + // COMPLETE is the recovery path: it has to rebuild L1 from the current base-table semantics, + // where every pre-existing row now reads the re-added column's default value. Whether the + // strict INCREMENTAL in between is rejected is a separate contract, owned by the suite for the + // baseline invalidation itself, so it is deliberately not asserted here -- this suite has to + // hold with or without that change. + cTask = refresh("cascade_l1", "COMPLETE") + assertEquals("SUCCESS", cTask.Status.toString(), "COMPLETE after schema ABA: " + cTask.ErrorMsg) + order_qt_cascade_l1_after_aba "SELECT grp, COUNT(*) AS c, SUM(amount) AS s FROM cascade_l1 GROUP BY grp" + + // ... and the downstream level stays refreshable afterwards. + cTask = refresh("cascade_l2", "COMPLETE") + assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 after L1 recovery: " + cTask.ErrorMsg) + order_qt_cascade_l2_after_aba "SELECT grp, row_count, total_amount FROM cascade_l2" +}