Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.analysis.ToSqlParams;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.BinlogConfig;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
Expand Down Expand Up @@ -625,6 +626,14 @@ private static MTMVAnalyzeQueryInfo analyzeQueryInternal(ConnectContext ctx, Map
properties = CreateTableInfo.addOlapHiddenColumns(
columns, isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS,
isIvm, properties, false);
// A row-binlog table carries hidden columns on top of the OLAP ones above, added by
// InternalCatalog#createOlapTable just before the table is built. The analyzed list has
// to carry them too: an MTMV re-validates its schema against it whenever a base table
// changes (MTMVPlanUtil#checkColumnIfChange), and a column missing here is
// indistinguishable from a real schema change. Idempotent, so the table still gets one.
CreateTableInfo.addRowBinlogHiddenColumns(columns,
isIvm ? KeysType.UNIQUE_KEYS : KeysType.DUP_KEYS, isIvm,
BinlogConfig.fromProperties(properties));
// analyze column
final boolean finalEnableMergeOnWrite = isIvm;
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1646,14 +1646,34 @@ public void checkLegalityOfPartitionExprs(PartitionTableInfo partitionTableInfo)
* Add hidden columns required by row binlog.
*/
public void createRowBinlogHiddenColumnsIfNecessary(BinlogConfig binlogConfig) {
if (!binlogConfig.isRowFormat()) {
addRowBinlogHiddenColumns(columns, keysType, isEnableMergeOnWrite, binlogConfig);
}

/**
* Append the hidden columns a row-binlog table carries. Callers that build the column list
* outside the create-table flow (an analyzed MTMV schema) go through here as well, so both
* sides of {@code MTMVPlanUtil#checkColumnIfChange} agree on the physical layout.
*
* <p>Idempotent: a column that is already present is kept once.
*/
public static void addRowBinlogHiddenColumns(List<ColumnDefinition> columns, KeysType keysType,
boolean isEnableMergeOnWrite, BinlogConfig binlogConfig) {
if (binlogConfig == null || !binlogConfig.isRowFormat()) {
return;
}
if (keysType.equals(KeysType.DUP_KEYS)) {
columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
columns.add(ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE));
addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
addIfAbsent(columns, ColumnDefinition.newRowLsnColumnDefinition(AggregateType.NONE));
} else if (keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite) {
columns.add(ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
addIfAbsent(columns, ColumnDefinition.newCommitTsoColumnDefinition(AggregateType.NONE));
}
}

private static void addIfAbsent(List<ColumnDefinition> columns, ColumnDefinition columnDefinition) {
boolean present = columns.stream()
.anyMatch(column -> column.getName().equalsIgnoreCase(columnDefinition.getName()));
if (!present) {
columns.add(columnDefinition);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,70 @@ public void testEnsureMTMVQueryUsableEnableIvmRewriteByRefreshMethod() throws Ex
incrementalCtx.getStatementContext().getIvmRewriteContext().orElseThrow().getMode());
}

@Test
public void testEnsureMTMVQueryUsableWithRowBinlogHiddenColumns() throws Exception {
// A table created with row binlog carries hidden columns its query never produces:
// __DORIS_COMMIT_TSO_COL__ for merge-on-write unique keys, plus __DORIS_ROW_LSN_COL__ for
// duplicate keys. InternalCatalog#createOlapTable adds them while the table is built, so an
// MV that enables row binlog itself ends up with them in its physical schema.
// ensureMTMVQueryUsable re-derives the schema from the query and compares the two
// (checkColumnIfChange), so the analyzed column list has to carry them too -- otherwise
// every refresh of such an MV fails with a spurious "column length not equals".
createTable("CREATE TABLE IF NOT EXISTS row_binlog_schema_base (\n"
+ " k1 int,\n"
+ " v1 int\n"
+ ")\n"
+ "DUPLICATE KEY(k1)\n"
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')\n");

createMvByNereids("create materialized view row_binlog_schema_ivm "
+ "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', "
+ "'binlog.format' = 'ROW') \n"
+ " as select k1, v1 from test.row_binlog_schema_base;");
createMvByNereids("create materialized view row_binlog_schema_dup "
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ " PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', "
+ "'binlog.format' = 'ROW') \n"
+ " as select k1, v1 from test.row_binlog_schema_base;");
createMvByNereids("create materialized view row_binlog_schema_plain "
+ "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ " PROPERTIES ('replication_num' = '1') \n"
+ " as select k1, v1 from test.row_binlog_schema_base;");

Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
MTMV ivmMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_ivm");
MTMV dupMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_dup");
MTMV plainMv = (MTMV) db.getTableOrAnalysisException("row_binlog_schema_plain");

Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL), rowBinlogHiddenColumns(ivmMv));
Assertions.assertEquals(Lists.newArrayList(Column.COMMIT_TSO_COL, Column.ROW_LSN_COL),
rowBinlogHiddenColumns(dupMv));
Assertions.assertTrue(rowBinlogHiddenColumns(plainMv).isEmpty());

for (MTMV mtmv : Lists.newArrayList(ivmMv, dupMv, plainMv)) {
ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv,
MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE);
Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, ctx),
"analyzed schema must match the physical schema of " + mtmv.getName());
}
}

private static List<String> rowBinlogHiddenColumns(MTMV mtmv) {
List<String> hidden = Lists.newArrayList();
for (Column column : mtmv.getBaseSchema(true)) {
if (column.getName().equalsIgnoreCase(Column.COMMIT_TSO_COL)
|| column.getName().equalsIgnoreCase(Column.ROW_LSN_COL)) {
hidden.add(column.getName().toUpperCase());
}
}
return hidden;
}

@Test
public void testEnsureMTMVQueryAnalyzeFailed() throws Exception {
createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !rb_ivm_rows --
1 10
2 20

-- !rb_ivm_no_binlog_rows --
1 10
2 20

-- !rb_dup_rows --
1 10
2 20

-- !rb_dup_no_binlog_rows --
1 10
2 20

-- !cascade_l2_baseline --
10 2 300
20 1 300

-- !cascade_l1_after_aba --
0 3 600

-- !cascade_l2_after_aba --
0 3 600

Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

import org.awaitility.Awaitility
import static java.util.concurrent.TimeUnit.SECONDS

// A materialized view re-validates its schema against a fresh analysis of its own query whenever a
// base table changes (MTMVPlanUtil.ensureMTMVQueryUsable -> checkColumnIfChange). A table created
// with row binlog carries hidden columns that the query never produces, and an MV carries them too
// when its own properties enable row binlog (which cascade IVM requires). The analyzed schema has to
// contain them as well, otherwise every refresh of such an MV fails with
// "column length not equals, please check whether columns of base table have changed" -- including a
// COMPLETE refresh, which is the only way out of a stale baseline.
//
// Covered here: the four MV shapes, and the DORIS-28306 shape where a cascade L1 could not recover
// through COMPLETE after a schema ABA on a referenced column.
suite("test_ivm_row_binlog_schema_validation") {
def rowBinlogProps = "'replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW'"
def plainProps = "'replication_num' = '1'"
// A cascade source keeps historical values so its own downstream MV can read the binlog.
def cascadeProps = rowBinlogProps + ", 'binlog.need_historical_value' = 'true'"

def ddlJobCount = { String table ->
return sql("""SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'""").size()
}

// `SHOW ALTER TABLE COLUMN` keeps finished jobs, so wait for a *new* job that is FINISHED.
def waitDdlFinished = { String table, int previousJobCount ->
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
def jobs = sql """SHOW ALTER TABLE COLUMN WHERE TableName = '${table}'"""
return jobs.size() > previousJobCount
&& jobs.every({ row -> row[9].toString() == 'FINISHED' })
})
}

def lastTaskId = null
// tasks('type'='mv') can briefly miss the just-finished task, so require a *new* TaskId.
def waitTerminalTask = { String mv ->
def taskResult
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
taskResult = sql_return_maparray("""
SELECT TaskId, Status, RefreshMode, ErrorMsg
FROM tasks('type'='mv')
WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mv}'
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
""")
return !taskResult.isEmpty()
&& taskResult[0].TaskId.toString() != lastTaskId
&& taskResult[0].Status.toString() != 'PENDING'
&& taskResult[0].Status.toString() != 'RUNNING'
})
lastTaskId = taskResult[0].TaskId.toString()
return taskResult[0]
}

def refresh = { String mv, String mode ->
sql """REFRESH MATERIALIZED VIEW ${mv} ${mode}"""
return waitTerminalTask(mv)
}

// ---------------------------------------------------------------- 1. four MV shapes
// Same query and same base table for all four; what differs is whether the MV itself enables
// row binlog, and (for the IVM ones) whether the MV is a MOW unique table.
def cases = [
[name: "rb_ivm", mvProps: rowBinlogProps, ivm: true],
[name: "rb_ivm_no_binlog", mvProps: plainProps, ivm: true],
[name: "rb_dup", mvProps: rowBinlogProps, ivm: false],
[name: "rb_dup_no_binlog", mvProps: plainProps, ivm: false],
]

for (def c : cases) {
def table = c.name + "_base"
def mv = c.name + "_mv"

sql """DROP MATERIALIZED VIEW IF EXISTS ${mv}"""
sql """DROP TABLE IF EXISTS ${table}"""
sql """
CREATE TABLE ${table} (
k1 INT NOT NULL,
v1 INT NULL,
spare INT NULL
)
UNIQUE KEY(k1)
DISTRIBUTED BY HASH(k1) BUCKETS 1
PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true',
'binlog.enable' = 'true', 'binlog.format' = 'ROW',
'binlog.need_historical_value' = 'true')
"""
sql """INSERT INTO ${table} VALUES (1, 10, 1), (2, 20, 1)"""

sql """
CREATE MATERIALIZED VIEW ${mv}
BUILD DEFERRED REFRESH ${c.ivm ? 'INCREMENTAL' : 'COMPLETE'} ON MANUAL
KEY(k1)
DISTRIBUTED BY HASH(k1) BUCKETS 1
PROPERTIES (${c.mvProps})
AS SELECT k1, SUM(v1) AS total FROM ${table} GROUP BY k1
"""
def task = refresh(mv, "COMPLETE")
assertEquals("SUCCESS", task.Status.toString(), "${c.name} baseline refresh: " + task.ErrorMsg)

// A schema change that leaves the query intact: the MV must stay refreshable. The dropped
// column is not referenced, so nothing about the MV's result changes.
def before = ddlJobCount(table)
sql """ALTER TABLE ${table} DROP COLUMN spare"""
waitDdlFinished(table, before)

task = refresh(mv, c.ivm ? "INCREMENTAL" : "COMPLETE")
assertEquals("SUCCESS", task.Status.toString(),
"${c.name} refresh after an unrelated DROP COLUMN: " + task.ErrorMsg)
}

// The dropped column was never referenced, so all four MVs must still hold their baseline rows.
order_qt_rb_ivm_rows "SELECT k1, total FROM rb_ivm_mv"
order_qt_rb_ivm_no_binlog_rows "SELECT k1, total FROM rb_ivm_no_binlog_mv"
order_qt_rb_dup_rows "SELECT k1, total FROM rb_dup_mv"
order_qt_rb_dup_no_binlog_rows "SELECT k1, total FROM rb_dup_no_binlog_mv"

// ------------------------------------- 2. DORIS-28306: cascade L1 must recover via COMPLETE
// A cascade forces row binlog onto L1, so L1 is exactly the shape that used to make every
// refresh of L1 fail after a schema change. An ABA on a referenced column must still leave the
// explicit COMPLETE recovery path usable.
def cTable = "cascade_aba_base"
sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l2"""
sql """DROP MATERIALIZED VIEW IF EXISTS cascade_l1"""
sql """DROP TABLE IF EXISTS ${cTable}"""
sql """
CREATE TABLE ${cTable} (
id BIGINT NOT NULL,
grp INT NULL,
amount BIGINT NULL
)
UNIQUE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true',
'binlog.enable' = 'true', 'binlog.format' = 'ROW',
'binlog.need_historical_value' = 'true')
"""
sql """INSERT INTO ${cTable} VALUES (1, 10, 100), (2, 10, 200), (3, 20, 300)"""

sql """
CREATE MATERIALIZED VIEW cascade_l1
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(id)
PROPERTIES (${cascadeProps})
AS SELECT id, grp, amount FROM ${cTable}
"""
def cTask = refresh("cascade_l1", "COMPLETE")
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L1 baseline: " + cTask.ErrorMsg)

sql """
CREATE MATERIALIZED VIEW cascade_l2
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL KEY(grp)
PROPERTIES (${cascadeProps})
AS SELECT grp, COUNT(*) AS row_count, SUM(amount) AS total_amount FROM cascade_l1 GROUP BY grp
"""
cTask = refresh("cascade_l2", "COMPLETE")
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 baseline: " + cTask.ErrorMsg)
order_qt_cascade_l2_baseline "SELECT grp, row_count, total_amount FROM cascade_l2"

def cBefore = ddlJobCount(cTable)
sql """ALTER TABLE ${cTable} DROP COLUMN grp"""
waitDdlFinished(cTable, cBefore)
cBefore = ddlJobCount(cTable)
sql """ALTER TABLE ${cTable} ADD COLUMN grp INT NULL DEFAULT '0'"""
waitDdlFinished(cTable, cBefore)

// COMPLETE is the recovery path: it has to rebuild L1 from the current base-table semantics,
// where every pre-existing row now reads the re-added column's default value. Whether the
// strict INCREMENTAL in between is rejected is a separate contract, owned by the suite for the
// baseline invalidation itself, so it is deliberately not asserted here -- this suite has to
// hold with or without that change.
cTask = refresh("cascade_l1", "COMPLETE")
assertEquals("SUCCESS", cTask.Status.toString(), "COMPLETE after schema ABA: " + cTask.ErrorMsg)
order_qt_cascade_l1_after_aba "SELECT grp, COUNT(*) AS c, SUM(amount) AS s FROM cascade_l1 GROUP BY grp"

// ... and the downstream level stays refreshable afterwards.
cTask = refresh("cascade_l2", "COMPLETE")
assertEquals("SUCCESS", cTask.Status.toString(), "cascade L2 after L1 recovery: " + cTask.ErrorMsg)
order_qt_cascade_l2_after_aba "SELECT grp, row_count, total_amount FROM cascade_l2"
}
Loading