diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java index 73f53608f58d03..f852d7bf13743a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java @@ -196,7 +196,10 @@ public String toSql(int depth) { } sb.append("PROPERTIES ("); - if (enableVariantDocMode) { + if (enableNestedGroup) { + // A nested group column has no subcolumn or doc mode settings, the parser clears them. + sb.append("\"variant_enable_nested_group\" = \"true\""); + } else if (enableVariantDocMode) { sb.append("\"variant_enable_doc_mode\" = \"") .append(String.valueOf(enableVariantDocMode)).append("\""); sb.append(","); @@ -219,11 +222,6 @@ public String toSql(int depth) { sb.append("\"variant_sparse_hash_shard_count\" = \"") .append(String.valueOf(Math.max(1, variantSparseHashShardCount))).append("\""); } - if (enableNestedGroup) { - sb.append(","); - sb.append("\"variant_enable_nested_group\" = \"") - .append(String.valueOf(enableNestedGroup)).append("\""); - } sb.append(")>"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c3c3fc704000de..b3bbdd1f113dc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1111,6 +1111,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.SqlModeHelper; import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta; import org.apache.doris.resource.workloadschedpolicy.WorkloadConditionMeta; @@ -1166,6 +1167,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { private static String DEFAULT_NESTED_COLUMN_NAME = "unnest"; private static String DEFAULT_ORDINALITY_COLUMN_NAME = "ordinality"; + // The built-in values of the default_variant_* session variables. + private static final SessionVariable BUILT_IN_SESSION_VARIABLE = new SessionVariable(); private static class ColumnDefinitionWithPath { private final ColumnDefinition columnDefinition; @@ -5400,25 +5403,25 @@ public DataType visitVariantPredefinedFields(VariantPredefinedFieldsContext ctx) Map properties = variantCtx.properties != null ? Maps.newHashMap(visitPropertyClause(variantCtx.properties)) : Maps.newHashMap(); - int variantMaxSubcolumnsCount = ConnectContext.get() == null ? 0 : - ConnectContext.get().getSessionVariable().getDefaultVariantMaxSubcolumnsCount(); - boolean enableTypedPathsToSparse = ConnectContext.get() == null ? false : - ConnectContext.get().getSessionVariable().getDefaultEnableTypedPathsToSparse(); - int variantMaxSparseColumnStatisticsSize = ConnectContext.get() == null ? 0 : - ConnectContext.get().getSessionVariable().getDefaultVariantMaxSparseColumnStatisticsSize(); + // The default_variant_* session variables only drive the tests, which run with debug points enabled. + // Elsewhere, and without a session, the properties a DDL leaves out take the built-in values, whatever the + // session holds (a global value set before the variables became test-only still loads from an image), so a + // DDL text, such as the one CREATE TABLE LIKE prints and parses, creates the same column in every session. + SessionVariable variantDefaults = BUILT_IN_SESSION_VARIABLE; + if (Config.enable_debug_points && ConnectContext.get() != null) { + variantDefaults = ConnectContext.get().getSessionVariable(); + } + int variantMaxSubcolumnsCount = variantDefaults.getDefaultVariantMaxSubcolumnsCount(); + boolean enableTypedPathsToSparse = variantDefaults.getDefaultEnableTypedPathsToSparse(); + int variantMaxSparseColumnStatisticsSize = variantDefaults.getDefaultVariantMaxSparseColumnStatisticsSize(); // default no bucketing - int variantSparseHashShardCount = ConnectContext.get() == null ? 0 : - ConnectContext.get().getSessionVariable().getDefaultVariantSparseHashShardCount(); - - boolean enableVariantDocMode = ConnectContext.get() == null ? false : - ConnectContext.get().getSessionVariable().getDefaultVariantEnableDocMode(); - long variantDocMaterializationMinRows = ConnectContext.get() == null ? 0L : - ConnectContext.get().getSessionVariable().getDefaultVariantDocMaterializationMinRows(); - int variantDocHashShardCount = ConnectContext.get() == null ? 128 : - ConnectContext.get().getSessionVariable().getDefaultVariantDocHashShardCount(); - - boolean enableNestedGroup = ConnectContext.get() == null ? true : - ConnectContext.get().getSessionVariable().getDefaultVariantEnableNestedGroup(); + int variantSparseHashShardCount = variantDefaults.getDefaultVariantSparseHashShardCount(); + + boolean enableVariantDocMode = variantDefaults.getDefaultVariantEnableDocMode(); + long variantDocMaterializationMinRows = variantDefaults.getDefaultVariantDocMaterializationMinRows(); + int variantDocHashShardCount = variantDefaults.getDefaultVariantDocHashShardCount(); + + boolean enableNestedGroup = variantDefaults.getDefaultVariantEnableNestedGroup(); try { // validate properties: variant_enable_doc_mode cannot be set together with other properties diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java index ad52a15ef8eec1..3fa8fb618036ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java @@ -204,7 +204,10 @@ public String toSql() { } sb.append("PROPERTIES ("); - if (enableVariantDocMode) { + if (enableNestedGroup) { + // A nested group column has no subcolumn or doc mode settings, the parser clears them. + sb.append("\"variant_enable_nested_group\" = \"true\""); + } else if (enableVariantDocMode) { sb.append("\"variant_enable_doc_mode\" = \"") .append(String.valueOf(enableVariantDocMode)).append("\""); sb.append(","); @@ -229,11 +232,6 @@ public String toSql() { .append(String.valueOf(Math.max(1, variantSparseHashShardCount))) .append("\""); } - if (enableNestedGroup) { - sb.append(","); - sb.append("\"variant_enable_nested_group\" = \"") - .append(String.valueOf(enableNestedGroup)).append("\""); - } sb.append(")>"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index ff315d8674fa69..46cfb087659782 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3901,6 +3901,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VariableMgr.VarAttr( name = DEFAULT_VARIANT_MAX_SUBCOLUMNS_COUNT, + flag = VariableMgr.INVISIBLE, needForward = true, checker = "checkDefaultVariantMaxSubcolumnsCount", fuzzy = true @@ -3909,13 +3910,16 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VariableMgr.VarAttr( name = DEFAULT_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public boolean defaultEnableTypedPathsToSparse = false; @VariableMgr.VarAttr( name = DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE, + flag = VariableMgr.INVISIBLE, needForward = true, checker = "checkDefaultVariantMaxSparseColumnStatisticsSize", fuzzy = true @@ -3933,7 +3937,9 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VariableMgr.VarAttr( name = DEFAULT_VARIANT_SPARSE_HASH_SHARD_COUNT, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public int defaultVariantSparseHashShardCount = 0; @@ -3946,7 +3952,9 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VariableMgr.VarAttr( name = DEFAULT_VARIANT_ENABLE_DOC_MODE, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public boolean defaultVariantEnableDocMode = false; @@ -3954,21 +3962,27 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VariableMgr.VarAttr( name = DEFAULT_VARIANT_DOC_MATERIALIZATION_MIN_ROWS, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public long defaultVariantDocMaterializationMinRows = 0L; @VariableMgr.VarAttr( name = DEFAULT_VARIANT_DOC_HASH_SHARD_COUNT, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public int defaultVariantDocHashShardCount = 64; @VariableMgr.VarAttr( name = DEFAULT_VARIANT_ENABLE_NESTED_GROUP, + flag = VariableMgr.INVISIBLE, needForward = true, + checker = "checkDefaultVariantSettable", fuzzy = true ) public boolean defaultVariantEnableNestedGroup = false; @@ -6994,7 +7008,17 @@ public static boolean enableStrictCast() { } } + // The default_variant_* variables only drive the tests, which run with debug points enabled. Users state VARIANT + // properties in the column definition, so a DDL text creates the same column in every session. + public void checkDefaultVariantSettable(String value) { + if (!Config.enable_debug_points) { + throw new UnsupportedOperationException("default_variant_* session variables are only for testing, " + + "set the property in the VARIANT column definition"); + } + } + public void checkDefaultVariantMaxSubcolumnsCount(String variantMaxSubcolumnsCount) { + checkDefaultVariantSettable(variantMaxSubcolumnsCount); int value = Integer.valueOf(variantMaxSubcolumnsCount); if (value < 0 || value > 100000) { throw new UnsupportedOperationException( @@ -7003,6 +7027,7 @@ public void checkDefaultVariantMaxSubcolumnsCount(String variantMaxSubcolumnsCou } public void checkDefaultVariantMaxSparseColumnStatisticsSize(String variantMaxSparseColumnStatisticsSize) { + checkDefaultVariantSettable(variantMaxSparseColumnStatisticsSize); int value = Integer.valueOf(variantMaxSparseColumnStatisticsSize); if (value < 1 || value > 50000) { throw new UnsupportedOperationException("variant max sparse column statistics size is: " diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java index 4368ff70c441fa..3a56a6fb382e17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; @@ -26,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.utframe.TestWithFeService; import org.apache.doris.utframe.UtFrameUtils; @@ -89,6 +91,44 @@ private static void createTable(String sql) throws Exception { } } + @Test + public void testVariantTypeIgnoresSessionDefaults() throws Exception { + SessionVariable sessionVariable = connectContext.getSessionVariable(); + boolean originalEnableDebugPoints = Config.enable_debug_points; + boolean originalDocMode = sessionVariable.defaultVariantEnableDocMode; + boolean originalNestedGroup = sessionVariable.defaultVariantEnableNestedGroup; + try { + Config.enable_debug_points = false; + createTable("CREATE TABLE test.variant_like_src (id INT, " + + "v VARIANT<'a.b':INT, PROPERTIES(\"variant_max_subcolumns_count\"=\"3\")>, " + + "z VARIANT, " + + "d VARIANT, " + + "n VARIANT) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")"); + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + Table source = db.getTableOrDdlException("variant_like_src"); + String sourceDdl = getCreateTableStmt(source); + // CREATE TABLE LIKE parses the DDL of the source, which prints neither a disabled doc mode nor a + // disabled nested group. Without debug points the session variables do not fill them in. + for (boolean docMode : new boolean[] {true, false}) { + sessionVariable.defaultVariantEnableDocMode = docMode; + sessionVariable.defaultVariantEnableNestedGroup = !docMode; + String targetName = docMode ? "variant_like_doc_mode" : "variant_like_nested_group"; + createTableLike("CREATE TABLE test." + targetName + " LIKE test.variant_like_src"); + Table target = db.getTableOrDdlException(targetName); + for (String columnName : new String[] {"v", "z", "d", "n"}) { + Assert.assertEquals(source.getColumn(columnName).getType(), + target.getColumn(columnName).getType()); + } + Assert.assertEquals(sourceDdl.replace("variant_like_src", targetName), getCreateTableStmt(target)); + } + } finally { + Config.enable_debug_points = originalEnableDebugPoints; + sessionVariable.defaultVariantEnableDocMode = originalDocMode; + sessionVariable.defaultVariantEnableNestedGroup = originalNestedGroup; + } + } + private static void createTableLike(String sql) throws Exception { NereidsParser nereidsParser = new NereidsParser(); CreateTableLikeCommand command = (CreateTableLikeCommand) nereidsParser.parseSingle(sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index 7f1861c96d2183..f8018b2eda5437 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.qe.SessionVariable; import org.junit.Assert; import org.junit.Test; @@ -172,7 +173,10 @@ public void testVariantToSqlSerializesNestedGroupProperty() { VariantType variantType = new VariantType(new ArrayList<>(), 0, false, 10000, 0, false, 0L, 64, true); - Assert.assertTrue(variantType.toSql().contains("\"variant_enable_nested_group\" = \"true\"")); + // Only the nested group property: the others are cleared for nested group and some, such as a statistics + // size of 0, would not parse. + Assert.assertEquals("variant", + variantType.toSql()); } @Test @@ -206,6 +210,20 @@ public void testVariantFieldCommentToSqlRoundTrip() { Assert.assertEquals(comment, reparsed.getPredefinedFields().get(0).getComment()); } + @Test + public void testVariantTakesBuiltInDefaults() { + // Without debug points, and without a session, the properties a DDL leaves out take the built-in values + // of the default_variant_* session variables. + org.apache.doris.nereids.types.VariantType parsed = (org.apache.doris.nereids.types.VariantType) + new NereidsParser().parseDataType("variant"); + SessionVariable builtIn = new SessionVariable(); + Assert.assertEquals(builtIn.defaultVariantMaxSubcolumnsCount, parsed.getVariantMaxSubcolumnsCount()); + Assert.assertEquals(builtIn.defaultVariantMaxSparseColumnStatisticsSize, + parsed.getVariantMaxSparseColumnStatisticsSize()); + Assert.assertEquals(builtIn.defaultVariantEnableDocMode, parsed.getEnableVariantDocMode()); + Assert.assertEquals(builtIn.defaultVariantEnableNestedGroup, parsed.getEnableNestedGroup()); + } + // ===================== Mixed Nesting & Precision ===================== @Test public void testArrayMapStructCombinationWithPrecision() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 0f8a4d1ccd5981..47c5cae946ca2f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -474,9 +474,17 @@ public void testAnalyzeVariantMaxSparseColumnStatisticsSize() throws AnalysisExc @Test public void testCheckDefaultVariantMaxSparseColumnStatisticsSize() { - SessionVariable sessionVariable = new SessionVariable(); - Assertions.assertThrows(UnsupportedOperationException.class, - () -> sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("0")); - sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("1"); + // The default_variant_* variables can only be set with debug points enabled. + boolean originalEnableDebugPoints = Config.enable_debug_points; + try { + Config.enable_debug_points = true; + SessionVariable sessionVariable = new SessionVariable(); + UnsupportedOperationException e = Assertions.assertThrows(UnsupportedOperationException.class, + () -> sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("0")); + Assertions.assertTrue(e.getMessage().contains("it must between 1 and 50000"), e.getMessage()); + sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("1"); + } finally { + Config.enable_debug_points = originalEnableDebugPoints; + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index c91d01b292820c..3f1ac7d24796bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -57,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.ReplayCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; @@ -73,6 +74,7 @@ import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.SqlModeHelper; import org.apache.doris.qe.StmtExecutor; @@ -1534,6 +1536,51 @@ public void testCreateTableVariantNestedGroupPropertyConflictsWithDocMode() { .contains("variant_enable_nested_group and variant_enable_doc_mode cannot both be true")); } + @Test + public void testCreateTableVariantSessionDefaultsOnlyWithDebugPoints() { + SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); + boolean originalEnableDebugPoints = Config.enable_debug_points; + boolean docMode = sessionVariable.defaultVariantEnableDocMode; + int count = sessionVariable.defaultVariantMaxSubcolumnsCount; + String sql = "CREATE TABLE t_variant (k1 INT, v VARIANT<'a':INT>, n VARIANT) DISTRIBUTED BY HASH(k1) BUCKETS 1"; + try { + sessionVariable.defaultVariantEnableDocMode = false; + sessionVariable.defaultVariantMaxSubcolumnsCount = 7; + Config.enable_debug_points = false; + // Without debug points a VARIANT column takes the built-in values, whatever the session holds. + List columns = ((CreateTableCommand) new NereidsParser().parseSingle(sql)) + .getCreateTableInfo().getColumnDefinitions(); + for (int i = 1; i < 3; i++) { + org.apache.doris.nereids.types.VariantType type = + (org.apache.doris.nereids.types.VariantType) columns.get(i).getType(); + Assertions.assertFalse(type.getEnableVariantDocMode()); + Assertions.assertEquals(new SessionVariable().defaultVariantMaxSubcolumnsCount, + type.getVariantMaxSubcolumnsCount()); + } + sessionVariable.defaultVariantEnableDocMode = true; + columns = ((CreateTableCommand) new NereidsParser().parseSingle(sql)) + .getCreateTableInfo().getColumnDefinitions(); + Assertions.assertFalse(((org.apache.doris.nereids.types.VariantType) columns.get(2).getType()) + .getEnableVariantDocMode()); + + // The tests run with debug points enabled and use the session variables. + Config.enable_debug_points = true; + columns = ((CreateTableCommand) new NereidsParser().parseSingle(sql)) + .getCreateTableInfo().getColumnDefinitions(); + Assertions.assertTrue(((org.apache.doris.nereids.types.VariantType) columns.get(2).getType()) + .getEnableVariantDocMode()); + sessionVariable.defaultVariantEnableDocMode = false; + columns = ((CreateTableCommand) new NereidsParser().parseSingle(sql)) + .getCreateTableInfo().getColumnDefinitions(); + Assertions.assertEquals(7, ((org.apache.doris.nereids.types.VariantType) columns.get(2).getType()) + .getVariantMaxSubcolumnsCount()); + } finally { + Config.enable_debug_points = originalEnableDebugPoints; + sessionVariable.defaultVariantEnableDocMode = docMode; + sessionVariable.defaultVariantMaxSubcolumnsCount = count; + } + } + @Test public void testCreateViewWithoutAs() { NereidsParser parser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java index 5219437fdc1a17..20f8d849c8ac39 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java @@ -35,9 +35,12 @@ import org.apache.doris.nereids.trees.plans.commands.SetOptionsCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.persist.GlobalVarPersistInfo; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.utframe.UtFrameUtils; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Assert; @@ -45,6 +48,10 @@ import org.junit.Test; import org.junit.jupiter.api.Assertions; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; import java.util.UUID; @@ -224,6 +231,69 @@ public void testAutoCommitType() throws AnalysisException { Assert.assertEquals(BigIntType.INSTANCE, l.getDataType()); } + @Test + public void testDefaultVariantVariablesOnlyForTesting() throws Exception { + boolean originalEnableDebugPoints = Config.enable_debug_points; + try { + for (String name : new String[] {SessionVariable.DEFAULT_VARIANT_MAX_SUBCOLUMNS_COUNT, + SessionVariable.DEFAULT_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE, + SessionVariable.DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE, + SessionVariable.DEFAULT_VARIANT_SPARSE_HASH_SHARD_COUNT, + SessionVariable.DEFAULT_VARIANT_ENABLE_DOC_MODE, + SessionVariable.DEFAULT_VARIANT_DOC_MATERIALIZATION_MIN_ROWS, + SessionVariable.DEFAULT_VARIANT_DOC_HASH_SHARD_COUNT, + SessionVariable.DEFAULT_VARIANT_ENABLE_NESTED_GROUP}) { + Assertions.assertTrue(VariableMgr.dump(SetType.SESSION, new SessionVariable(), null).stream() + .noneMatch(row -> row.get(0).equals(name)), name); + + SessionVariable sessionVariable = new SessionVariable(); + String defaultValue = sessionVariable.getForwardVariables().get(name); + Config.enable_debug_points = false; + for (SetType setType : new SetType[] {SetType.SESSION, SetType.GLOBAL}) { + SetVar setVar = new SetVar(setType, name, new StringLiteral("1")); + DdlException e = Assertions.assertThrows(DdlException.class, + () -> VariableMgr.setVar(sessionVariable, setVar)); + Assertions.assertTrue(e.getMessage().contains("only for testing"), e.getMessage()); + } + Assertions.assertEquals(defaultValue, sessionVariable.getForwardVariables().get(name)); + Assertions.assertEquals(defaultValue, VariableMgr.newSessionVariable().getForwardVariables().get(name)); + + // A statement forwarded by another FE carries the value of its session + SessionVariable forwarded = new SessionVariable(); + forwarded.setForwardedSessionVariables(ImmutableMap.of(name, "1")); + String value = forwarded.getForwardVariables().get(name); + Assertions.assertNotEquals(defaultValue, value, name); + + Config.enable_debug_points = true; + VariableMgr.setVar(sessionVariable, new SetVar(SetType.SESSION, name, new StringLiteral("1"))); + Assertions.assertEquals(value, sessionVariable.getForwardVariables().get(name)); + } + } finally { + Config.enable_debug_points = originalEnableDebugPoints; + } + } + + @Test + public void testPersistedDefaultVariantVariableDroppedOnReplay() throws Exception { + boolean originalEnableDebugPoints = Config.enable_debug_points; + try { + Config.enable_debug_points = false; + // A value set by SET GLOBAL before the upgrade + SessionVariable persisted = new SessionVariable(); + persisted.defaultVariantEnableDocMode = true; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + new GlobalVarPersistInfo(persisted, Lists.newArrayList(SessionVariable.DEFAULT_VARIANT_ENABLE_DOC_MODE)) + .write(new DataOutputStream(bytes)); + GlobalVarPersistInfo info = GlobalVarPersistInfo.read( + new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + + VariableMgr.replayGlobalVariableV2(info, false); + Assertions.assertFalse(VariableMgr.newSessionVariable().defaultVariantEnableDocMode); + } finally { + Config.enable_debug_points = originalEnableDebugPoints; + } + } + @Test public void testCheckSqlConvertorFeatures() throws DdlException { // set wrong var diff --git a/regression-test/data/variant_p0/test_variant_create_table_like.out b/regression-test/data/variant_p0/test_variant_create_table_like.out new file mode 100644 index 00000000000000..eafe2e4468f4f6 --- /dev/null +++ b/regression-test/data/variant_p0/test_variant_create_table_like.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_src -- +id int Yes true \N +v variant Yes false \N NONE +d variant Yes false \N NONE +n variant Yes false \N NONE + +-- !desc_dst -- +id int Yes true \N +v variant Yes false \N NONE +d variant Yes false \N NONE +n variant Yes false \N NONE + diff --git a/regression-test/suites/variant_p0/test_variant_create_table_like.groovy b/regression-test/suites/variant_p0/test_variant_create_table_like.groovy new file mode 100644 index 00000000000000..7107956a50155b --- /dev/null +++ b/regression-test/suites/variant_p0/test_variant_create_table_like.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// CREATE TABLE LIKE prints the source as DDL text and parses it again. The text of a nested group VARIANT +// column used to carry variant_max_sparse_column_statistics_size = 0, which does not parse, so the statement +// failed. The properties the text leaves out, such as a disabled doc mode, come from the default_variant_* +// session variables, which take effect only with debug points enabled, as in the tests; the fuzzy runs pick +// them at random, so set them to their defaults. +suite("test_variant_create_table_like", "p0") { + sql """ set default_variant_enable_doc_mode = false """ + sql """ set default_variant_enable_nested_group = false """ + sql """ DROP TABLE IF EXISTS variant_create_like_src """ + sql """ + CREATE TABLE variant_create_like_src ( + id INT, + v VARIANT, + d VARIANT, + n VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + qt_desc_src """ desc variant_create_like_src """ + + sql """ DROP TABLE IF EXISTS variant_create_like_dst """ + sql """ CREATE TABLE variant_create_like_dst LIKE variant_create_like_src """ + qt_desc_dst """ desc variant_create_like_dst """ +}